{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "967acdf8-b5f2-4746-a56f-28fdfc27595d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "============GPU================\n",
      "Sat Dec 23 11:06:21 2023       \n",
      "+---------------------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 535.104.05             Driver Version: 535.104.05   CUDA Version: 12.2     |\n",
      "|-----------------------------------------+----------------------+----------------------+\n",
      "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                                         |                      |               MIG M. |\n",
      "|=========================================+======================+======================|\n",
      "|   0  NVIDIA GeForce RTX 3090        On  | 00000000:04:00.0 Off |                  N/A |\n",
      "| 30%   41C    P8              27W / 350W |      3MiB / 24576MiB |      0%      Default |\n",
      "|                                         |                      |                  N/A |\n",
      "+-----------------------------------------+----------------------+----------------------+\n",
      "                                                                                         \n",
      "+---------------------------------------------------------------------------------------+\n",
      "| Processes:                                                                            |\n",
      "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
      "|        ID   ID                                                             Usage      |\n",
      "|=======================================================================================|\n",
      "|  No running processes found                                                           |\n",
      "+---------------------------------------------------------------------------------------+\n",
      "============CUDA version================\n",
      "nvcc: NVIDIA (R) Cuda compiler driver\n",
      "Copyright (c) 2005-2022 NVIDIA Corporation\n",
      "Built on Wed_Sep_21_10:33:58_PDT_2022\n",
      "Cuda compilation tools, release 11.8, V11.8.89\n",
      "Build cuda_11.8.r11.8/compiler.31833905_0\n",
      "============CPU================\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "model name\t: Intel(R) Xeon(R) CPU E5-2696 v3 @ 2.30GHz\n",
      "============Memory================\n",
      "MemTotal:       131851148 kB\n"
     ]
    }
   ],
   "source": [
    "# GPU\n",
    "print(\"============GPU================\")\n",
    "!nvidia-smi\n",
    "\n",
    "# CUDA version\n",
    "print(\"============CUDA version================\")\n",
    "!nvcc --version\n",
    "\n",
    "# CPU\n",
    "print(\"============CPU================\")\n",
    "!cat /proc/cpuinfo | grep model\\ name\n",
    "\n",
    "# Memory\n",
    "print(\"============Memory================\")\n",
    "!cat /proc/meminfo | grep MemTotal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bb0c71b2-b7d2-47b2-82ab-24619929a13d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'llama'...\n",
      "remote: Enumerating objects: 417, done.\u001b[K\n",
      "remote: Counting objects: 100% (71/71), done.\u001b[K\n",
      "remote: Compressing objects: 100% (49/49), done.\u001b[K\n",
      "remote: Total 417 (delta 29), reused 48 (delta 14), pack-reused 346\u001b[K\n",
      "Receiving objects: 100% (417/417), 1.10 MiB | 10.41 MiB/s, done.\n",
      "Resolving deltas: 100% (214/214), done.\n"
     ]
    }
   ],
   "source": [
    "!git clone https://github.com/facebookresearch/llama"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "080f46e7-5783-4fc1-9552-59f9021bdfc7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspace/llama\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/IPython/core/magics/osm.py:417: UserWarning: using dhist requires you to install the `pickleshare` library.\n",
      "  self.shell.db['dhist'] = compress_dhist(dhist)[-100:]\n"
     ]
    }
   ],
   "source": [
    "%cd llama"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e764773d-63c6-4076-bc0b-79583e7927b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Downloading LICENSE and Acceptable Usage Policy\n",
      "--2023-12-21 18:46:25--  https://download.llamameta.net/LICENSE?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.52, 108.138.106.87, 108.138.106.23, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.52|:443... connected.\n",
      "HTTP request sent, awaiting response... 416 Requested Range Not Satisfiable\n",
      "\n",
      "    The file is already fully retrieved; nothing to do.\n",
      "\n",
      "--2023-12-21 18:46:25--  https://download.llamameta.net/USE_POLICY.md?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.87, 108.138.106.50, 108.138.106.52, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.87|:443... connected.\n",
      "HTTP request sent, awaiting response... 416 Requested Range Not Satisfiable\n",
      "\n",
      "    The file is already fully retrieved; nothing to do.\n",
      "\n",
      "Downloading tokenizer\n",
      "--2023-12-21 18:46:25--  https://download.llamameta.net/tokenizer.model?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.87, 108.138.106.52, 108.138.106.50, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.87|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 499723 (488K) [binary/octet-stream]\n",
      "Saving to: ‘./tokenizer.model’\n",
      "\n",
      "./tokenizer.model   100%[===================>] 488.01K  --.-KB/s    in 0.07s   \n",
      "\n",
      "2023-12-21 18:46:25 (7.11 MB/s) - ‘./tokenizer.model’ saved [499723/499723]\n",
      "\n",
      "--2023-12-21 18:46:25--  https://download.llamameta.net/tokenizer_checklist.chk?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.52, 108.138.106.50, 108.138.106.87, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.52|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 50 [binary/octet-stream]\n",
      "Saving to: ‘./tokenizer_checklist.chk’\n",
      "\n",
      "./tokenizer_checkli 100%[===================>]      50  --.-KB/s    in 0s      \n",
      "\n",
      "2023-12-21 18:46:26 (69.1 MB/s) - ‘./tokenizer_checklist.chk’ saved [50/50]\n",
      "\n",
      "tokenizer.model: OK\n",
      "Downloading llama-2-7b\n",
      "--2023-12-21 18:46:26--  https://download.llamameta.net/llama-2-7b/consolidated.00.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.52, 108.138.106.50, 108.138.106.23, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.52|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 13476925163 (13G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-7b/consolidated.00.pth’\n",
      "\n",
      "./llama-2-7b/consol 100%[===================>]  12.55G  35.9MB/s    in 5m 13s  \n",
      "\n",
      "2023-12-21 18:51:39 (41.1 MB/s) - ‘./llama-2-7b/consolidated.00.pth’ saved [13476925163/13476925163]\n",
      "\n",
      "--2023-12-21 18:51:39--  https://download.llamameta.net/llama-2-7b/params.json?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.50, 108.138.106.87, 108.138.106.52, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.50|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 102 [application/json]\n",
      "Saving to: ‘./llama-2-7b/params.json’\n",
      "\n",
      "./llama-2-7b/params 100%[===================>]     102  --.-KB/s    in 0s      \n",
      "\n",
      "2023-12-21 18:51:39 (10.1 MB/s) - ‘./llama-2-7b/params.json’ saved [102/102]\n",
      "\n",
      "--2023-12-21 18:51:39--  https://download.llamameta.net/llama-2-7b/checklist.chk?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.52, 108.138.106.50, 108.138.106.23, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.52|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 100 [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-7b/checklist.chk’\n",
      "\n",
      "./llama-2-7b/checkl 100%[===================>]     100  --.-KB/s    in 0s      \n",
      "\n",
      "2023-12-21 18:51:39 (119 MB/s) - ‘./llama-2-7b/checklist.chk’ saved [100/100]\n",
      "\n",
      "Checking checksums\n",
      "consolidated.00.pth: OK\n",
      "params.json: OK\n",
      "Downloading llama-2-13b\n",
      "--2023-12-21 18:52:04--  https://download.llamameta.net/llama-2-13b/consolidated.00.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.87, 108.138.106.52, 108.138.106.23, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.87|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 13016329643 (12G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-13b/consolidated.00.pth’\n",
      "\n",
      "./llama-2-13b/conso 100%[===================>]  12.12G  60.8MB/s    in 3m 59s  \n",
      "\n",
      "2023-12-21 18:56:03 (52.0 MB/s) - ‘./llama-2-13b/consolidated.00.pth’ saved [13016329643/13016329643]\n",
      "\n",
      "--2023-12-21 18:56:03--  https://download.llamameta.net/llama-2-13b/consolidated.01.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.50, 108.138.106.87, 108.138.106.52, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.50|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 13016329643 (12G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-13b/consolidated.01.pth’\n",
      "\n",
      "./llama-2-13b/conso 100%[===================>]  12.12G  71.7MB/s    in 4m 1s   \n",
      "\n",
      "2023-12-21 19:00:05 (51.6 MB/s) - ‘./llama-2-13b/consolidated.01.pth’ saved [13016329643/13016329643]\n",
      "\n",
      "--2023-12-21 19:00:05--  https://download.llamameta.net/llama-2-13b/params.json?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.50, 108.138.106.87, 108.138.106.23, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.50|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 102 [application/json]\n",
      "Saving to: ‘./llama-2-13b/params.json’\n",
      "\n",
      "./llama-2-13b/param 100%[===================>]     102  --.-KB/s    in 0s      \n",
      "\n",
      "2023-12-21 19:00:05 (12.3 MB/s) - ‘./llama-2-13b/params.json’ saved [102/102]\n",
      "\n",
      "--2023-12-21 19:00:05--  https://download.llamameta.net/llama-2-13b/checklist.chk?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.50, 108.138.106.23, 108.138.106.52, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.50|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 154 [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-13b/checklist.chk’\n",
      "\n",
      "./llama-2-13b/check 100%[===================>]     154  --.-KB/s    in 0s      \n",
      "\n",
      "2023-12-21 19:00:05 (229 MB/s) - ‘./llama-2-13b/checklist.chk’ saved [154/154]\n",
      "\n",
      "Checking checksums\n",
      "consolidated.00.pth: OK\n",
      "consolidated.01.pth: OK\n",
      "params.json: OK\n",
      "Downloading llama-2-70b\n",
      "--2023-12-21 19:00:53--  https://download.llamameta.net/llama-2-70b/consolidated.00.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.50, 108.138.106.23, 108.138.106.52, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.50|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 17246706245 (16G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-70b/consolidated.00.pth’\n",
      "\n",
      "./llama-2-70b/conso 100%[===================>]  16.06G  99.3MB/s    in 3m 12s  \n",
      "\n",
      "2023-12-21 19:04:05 (85.8 MB/s) - ‘./llama-2-70b/consolidated.00.pth’ saved [17246706245/17246706245]\n",
      "\n",
      "--2023-12-21 19:04:05--  https://download.llamameta.net/llama-2-70b/consolidated.01.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.23, 108.138.106.50, 108.138.106.52, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.23|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 17246706245 (16G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-70b/consolidated.01.pth’\n",
      "\n",
      "./llama-2-70b/conso 100%[===================>]  16.06G  73.7MB/s    in 3m 28s  \n",
      "\n",
      "2023-12-21 19:07:33 (79.1 MB/s) - ‘./llama-2-70b/consolidated.01.pth’ saved [17246706245/17246706245]\n",
      "\n",
      "--2023-12-21 19:07:33--  https://download.llamameta.net/llama-2-70b/consolidated.02.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.50, 108.138.106.23, 108.138.106.52, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.50|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 17246706245 (16G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-70b/consolidated.02.pth’\n",
      "\n",
      "./llama-2-70b/conso 100%[===================>]  16.06G  97.8MB/s    in 3m 21s  \n",
      "\n",
      "2023-12-21 19:10:55 (81.7 MB/s) - ‘./llama-2-70b/consolidated.02.pth’ saved [17246706245/17246706245]\n",
      "\n",
      "--2023-12-21 19:10:55--  https://download.llamameta.net/llama-2-70b/consolidated.03.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.23, 108.138.106.50, 108.138.106.87, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.23|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 17246706245 (16G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-70b/consolidated.03.pth’\n",
      "\n",
      "./llama-2-70b/conso 100%[===================>]  16.06G  88.7MB/s    in 3m 39s  \n",
      "\n",
      "2023-12-21 19:14:34 (75.0 MB/s) - ‘./llama-2-70b/consolidated.03.pth’ saved [17246706245/17246706245]\n",
      "\n",
      "--2023-12-21 19:14:34--  https://download.llamameta.net/llama-2-70b/consolidated.04.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.87, 108.138.106.23, 108.138.106.50, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.87|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 17246706245 (16G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-70b/consolidated.04.pth’\n",
      "\n",
      "./llama-2-70b/conso 100%[===================>]  16.06G  9.11MB/s    in 3m 59s  \n",
      "\n",
      "2023-12-21 19:18:34 (68.8 MB/s) - ‘./llama-2-70b/consolidated.04.pth’ saved [17246706245/17246706245]\n",
      "\n",
      "--2023-12-21 19:18:34--  https://download.llamameta.net/llama-2-70b/consolidated.05.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.87, 108.138.106.23, 108.138.106.50, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.87|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 17246706245 (16G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-70b/consolidated.05.pth’\n",
      "\n",
      "./llama-2-70b/conso 100%[===================>]  16.06G  91.5MB/s    in 3m 9s   \n",
      "\n",
      "2023-12-21 19:21:42 (87.2 MB/s) - ‘./llama-2-70b/consolidated.05.pth’ saved [17246706245/17246706245]\n",
      "\n",
      "--2023-12-21 19:21:42--  https://download.llamameta.net/llama-2-70b/consolidated.06.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.52, 108.138.106.23, 108.138.106.50, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.52|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 17246706245 (16G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-70b/consolidated.06.pth’\n",
      "\n",
      "./llama-2-70b/conso 100%[===================>]  16.06G  88.5MB/s    in 3m 0s   \n",
      "\n",
      "2023-12-21 19:24:43 (91.3 MB/s) - ‘./llama-2-70b/consolidated.06.pth’ saved [17246706245/17246706245]\n",
      "\n",
      "--2023-12-21 19:24:43--  https://download.llamameta.net/llama-2-70b/consolidated.07.pth?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.87, 108.138.106.23, 108.138.106.52, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.87|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 17246706245 (16G) [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-70b/consolidated.07.pth’\n",
      "\n",
      "./llama-2-70b/conso 100%[===================>]  16.06G  46.3MB/s    in 4m 20s  \n",
      "\n",
      "2023-12-21 19:29:03 (63.2 MB/s) - ‘./llama-2-70b/consolidated.07.pth’ saved [17246706245/17246706245]\n",
      "\n",
      "--2023-12-21 19:29:03--  https://download.llamameta.net/llama-2-70b/params.json?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.23, 108.138.106.50, 108.138.106.52, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.23|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 147 [application/json]\n",
      "Saving to: ‘./llama-2-70b/params.json’\n",
      "\n",
      "./llama-2-70b/param 100%[===================>]     147  --.-KB/s    in 0s      \n",
      "\n",
      "2023-12-21 19:29:04 (1.68 MB/s) - ‘./llama-2-70b/params.json’ saved [147/147]\n",
      "\n",
      "--2023-12-21 19:29:04--  https://download.llamameta.net/llama-2-70b/checklist.chk?Policy=eyJTdGF0ZW1lbnQiOlt7InVuaXF1ZV9oYXNoIjoibW1ubGM4OTh2aHhtYjNwbnQzMWdvdmpzIiwiUmVzb3VyY2UiOiJodHRwczpcL1wvZG93bmxvYWQubGxhbWFtZXRhLm5ldFwvKiIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcwMzI3MDU4M319fV19&Signature=aQwZ2GgPZgHodvWADnPipLid%7EgO-8Sp6j58tAz8XL6iEUuaIKCfSrSdQXImxh%7EiM3Jjj3K8THvK%7Et-V8Nzu3MOAzwc9-FGdQJpsjku8JsRQXGFLR3HdeeXcghSP1LP1nkB59XN4gcTxBeLcxcTsX%7Eo9qeeG5Nxe2oheb7HeRDCHr90Ur7HKxKWLBbczl%7Er6RjXMhipS05rE3pgsQaiur99Zm8dlaMbON2CfSG6OhhBHNy1BTG%7EgNIzExXCREHAOethodX8gm9uc8CzCr95k3%7EZ0wNHRiGcSxz77UhkJLGYF2euZilsR-wsCsv9wRgVyUNtfA4h5Z%7ESq59Vtozuq9uA__&Key-Pair-Id=K15QRJLYKIFSLZ&Download-Request-ID=370498102131315\n",
      "Resolving download.llamameta.net (download.llamameta.net)... 108.138.106.87, 108.138.106.52, 108.138.106.23, ...\n",
      "Connecting to download.llamameta.net (download.llamameta.net)|108.138.106.87|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 478 [binary/octet-stream]\n",
      "Saving to: ‘./llama-2-70b/checklist.chk’\n",
      "\n",
      "./llama-2-70b/check 100%[===================>]     478  --.-KB/s    in 0s      \n",
      "\n",
      "2023-12-21 19:29:04 (38.1 MB/s) - ‘./llama-2-70b/checklist.chk’ saved [478/478]\n",
      "\n",
      "Checking checksums\n",
      "consolidated.00.pth: OK\n",
      "consolidated.01.pth: OK\n",
      "consolidated.02.pth: OK\n",
      "consolidated.03.pth: OK\n",
      "consolidated.04.pth: OK\n",
      "consolidated.05.pth: OK\n",
      "consolidated.06.pth: OK\n",
      "consolidated.07.pth: OK\n",
      "params.json: OK\n"
     ]
    }
   ],
   "source": [
    "# Define your PRESIGNED_URL and MODEL_SIZE in the script to prevent asking in the notebook\n",
    "!bash download.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1541dcc9-b822-4783-b6eb-020fc4a0316d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspace\n"
     ]
    }
   ],
   "source": [
    "%cd /workspace\n",
    "!mkdir -p llama.cpp/models/7B-v2/\n",
    "!mv llama/llama-2-7b/* llama.cpp/models/7B-v2/\n",
    "!mkdir -p llama.cpp/models/13B-v2/\n",
    "!mv llama/llama-2-13b/* llama.cpp/models/13B-v2/\n",
    "!mkdir -p llama.cpp/models/70B-v2/\n",
    "!mv llama/llama-2-70b/* llama.cpp/models/70B-v2/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cea2bab8-7c0d-42cc-8e32-064e71a58a74",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspace/llama.cpp\n"
     ]
    }
   ],
   "source": [
    "%cd llama.cpp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9594719a-ea2a-4b8d-bd26-2c19f0a6a2de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100 13283  100 13283    0     0  73817      0 --:--:-- --:--:-- --:--:-- 73794\n"
     ]
    }
   ],
   "source": [
    "# If you encounter the error \"does not appear to have a file named config.json\" when converting the models to ggml FP16 format, try to convert the model to huggingface format to get the config.json file.\n",
    "!curl -o convert_llama_weights_to_hf.py https://raw.githubusercontent.com/huggingface/transformers/main/src/transformers/models/llama/convert_llama_weights_to_hf.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "06b44ccf-4bf6-47a4-8f05-87a662822110",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspace/llama.cpp/models\n"
     ]
    }
   ],
   "source": [
    "%cd models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8f541028-baaa-40f8-8e1c-c359b5ead34c",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp tokenizer.model 7B-v2/\n",
    "!cp tokenizer.model 13B-v2/\n",
    "!cp tokenizer.model 70B-v2/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fcc944e7-32a8-4918-8001-6b51fb835377",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspace/llama.cpp\n"
     ]
    }
   ],
   "source": [
    "%cd .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ae14bc36-f1ba-4069-82bc-63242471a393",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting numpy==1.24.4 (from -r requirements.txt (line 1))\n",
      "  Using cached numpy-1.24.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.6 kB)\n",
      "Collecting sentencepiece==0.1.98 (from -r requirements.txt (line 2))\n",
      "  Using cached sentencepiece-0.1.98-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
      "Collecting transformers>=4.34.0 (from -r requirements.txt (line 3))\n",
      "  Using cached transformers-4.36.2-py3-none-any.whl.metadata (126 kB)\n",
      "Collecting gguf>=0.1.0 (from -r requirements.txt (line 4))\n",
      "  Using cached gguf-0.6.0-py3-none-any.whl.metadata (3.2 kB)\n",
      "Collecting protobuf>=4.21.0 (from -r requirements.txt (line 5))\n",
      "  Using cached protobuf-4.25.1-cp37-abi3-manylinux2014_x86_64.whl.metadata (541 bytes)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers>=4.34.0->-r requirements.txt (line 3)) (3.13.1)\n",
      "Collecting huggingface-hub<1.0,>=0.19.3 (from transformers>=4.34.0->-r requirements.txt (line 3))\n",
      "  Using cached huggingface_hub-0.20.1-py3-none-any.whl.metadata (12 kB)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.34.0->-r requirements.txt (line 3)) (23.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.34.0->-r requirements.txt (line 3)) (6.0.1)\n",
      "Collecting regex!=2019.12.17 (from transformers>=4.34.0->-r requirements.txt (line 3))\n",
      "  Using cached regex-2023.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (40 kB)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers>=4.34.0->-r requirements.txt (line 3)) (2.31.0)\n",
      "Collecting tokenizers<0.19,>=0.14 (from transformers>=4.34.0->-r requirements.txt (line 3))\n",
      "  Using cached tokenizers-0.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\n",
      "Collecting safetensors>=0.3.1 (from transformers>=4.34.0->-r requirements.txt (line 3))\n",
      "  Using cached safetensors-0.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.8 kB)\n",
      "Collecting tqdm>=4.27 (from transformers>=4.34.0->-r requirements.txt (line 3))\n",
      "  Using cached tqdm-4.66.1-py3-none-any.whl.metadata (57 kB)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers>=4.34.0->-r requirements.txt (line 3)) (2023.10.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers>=4.34.0->-r requirements.txt (line 3)) (4.8.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers>=4.34.0->-r requirements.txt (line 3)) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers>=4.34.0->-r requirements.txt (line 3)) (3.6)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers>=4.34.0->-r requirements.txt (line 3)) (2.1.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers>=4.34.0->-r requirements.txt (line 3)) (2023.11.17)\n",
      "Using cached numpy-1.24.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB)\n",
      "Using cached transformers-4.36.2-py3-none-any.whl (8.2 MB)\n",
      "Using cached gguf-0.6.0-py3-none-any.whl (23 kB)\n",
      "Using cached protobuf-4.25.1-cp37-abi3-manylinux2014_x86_64.whl (294 kB)\n",
      "Using cached huggingface_hub-0.20.1-py3-none-any.whl (330 kB)\n",
      "Using cached regex-2023.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (773 kB)\n",
      "Using cached safetensors-0.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
      "Using cached tokenizers-0.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.8 MB)\n",
      "Using cached tqdm-4.66.1-py3-none-any.whl (78 kB)\n",
      "Installing collected packages: sentencepiece, tqdm, safetensors, regex, protobuf, numpy, huggingface-hub, gguf, tokenizers, transformers\n",
      "  Attempting uninstall: numpy\n",
      "    Found existing installation: numpy 1.26.2\n",
      "    Uninstalling numpy-1.26.2:\n",
      "      Successfully uninstalled numpy-1.26.2\n",
      "Successfully installed gguf-0.6.0 huggingface-hub-0.20.1 numpy-1.24.4 protobuf-4.25.1 regex-2023.10.3 safetensors-0.4.1 sentencepiece-0.1.98 tokenizers-0.15.0 tqdm-4.66.1 transformers-4.36.2\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.3.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# !pip uninstall accelerate # If you have this package, uninstall it first, then use `convert to hf model` to get the config.json.\n",
    "# install Python dependencies\n",
    "!python3 -m pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a95b3a38-17fb-4792-b1cd-4255e6ed2a7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
      "Fetching all parameters from the checkpoint at models/7B-v2/.\n",
      "Loading the checkpoint in a Llama model.\n",
      "Traceback (most recent call last):\n",
      "  File \"/workspace/llama.cpp/convert_llama_weights_to_hf.py\", line 319, in <module>\n",
      "    main()\n",
      "  File \"/workspace/llama.cpp/convert_llama_weights_to_hf.py\", line 307, in main\n",
      "    write_model(\n",
      "  File \"/workspace/llama.cpp/convert_llama_weights_to_hf.py\", line 271, in write_model\n",
      "    model = LlamaForCausalLM.from_pretrained(tmp_model_path, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True)\n",
      "  File \"/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py\", line 2863, in from_pretrained\n",
      "    raise ImportError(\n",
      "ImportError: Using `low_cpu_mem_usage=True` or a `device_map` requires Accelerate: `pip install accelerate`\n",
      "You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
      "Fetching all parameters from the checkpoint at models/13B-v2/.\n",
      "Loading the checkpoint in a Llama model.\n",
      "Traceback (most recent call last):\n",
      "  File \"/workspace/llama.cpp/convert_llama_weights_to_hf.py\", line 319, in <module>\n",
      "    main()\n",
      "  File \"/workspace/llama.cpp/convert_llama_weights_to_hf.py\", line 307, in main\n",
      "    write_model(\n",
      "  File \"/workspace/llama.cpp/convert_llama_weights_to_hf.py\", line 271, in write_model\n",
      "    model = LlamaForCausalLM.from_pretrained(tmp_model_path, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True)\n",
      "  File \"/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py\", line 2863, in from_pretrained\n",
      "    raise ImportError(\n",
      "ImportError: Using `low_cpu_mem_usage=True` or a `device_map` requires Accelerate: `pip install accelerate`\n",
      "You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
      "Fetching all parameters from the checkpoint at models/70B-v2/.\n"
     ]
    }
   ],
   "source": [
    "# We don't need these models actually. We only need this to figure out the config.json error.\n",
    "!python3 convert_llama_weights_to_hf.py --input_dir models/7B-v2/ --model_size 7B --output_dir models/7B-v2/\n",
    "!python3 convert_llama_weights_to_hf.py --input_dir models/13B-v2/ --model_size 13B --output_dir models/13B-v2/\n",
    "!python3 convert_llama_weights_to_hf.py --input_dir models/70B-v2/ --model_size 70B --output_dir models/70B-v2/ # Surprisingly, it still solves the problem although you can't find the config.json file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "839375fa-44f5-498c-8f1e-0e22ad8311ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Edit your params.json file if the \"vocab_size\" mismatch\n",
    "import json\n",
    "\n",
    "# Load the JSON file\n",
    "with open('models/7B-v2/params.json', 'r') as file:\n",
    "    data = json.load(file)\n",
    "\n",
    "# Modify the 'vocab_size' key\n",
    "data['vocab_size'] = 32000\n",
    "\n",
    "# Write the modified data back to the file\n",
    "with open('models/7B-v2/params.json', 'w') as file:\n",
    "    json.dump(data, file, indent=4)\n",
    "\n",
    "# Load the JSON file\n",
    "with open('models/13B-v2/params.json', 'r') as file:\n",
    "    data = json.load(file)\n",
    "\n",
    "# Modify the 'vocab_size' key\n",
    "data['vocab_size'] = 32000\n",
    "\n",
    "# Write the modified data back to the file\n",
    "with open('models/13B-v2/params.json', 'w') as file:\n",
    "    json.dump(data, file, indent=4)\n",
    "\n",
    "# Load the JSON file\n",
    "with open('models/70B-v2/params.json', 'r') as file:\n",
    "    data = json.load(file)\n",
    "\n",
    "# Modify the 'vocab_size' key\n",
    "data['vocab_size'] = 32000\n",
    "\n",
    "# Write the modified data back to the file\n",
    "with open('models/70B-v2/params.json', 'w') as file:\n",
    "    json.dump(data, file, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c5ce9c63-6f03-4736-a1df-56b9605f698b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading model file models/7B-v2/consolidated.00.pth\n",
      "params = Params(n_vocab=32000, n_embd=4096, n_layer=32, n_ctx=4096, n_ff=11008, n_head=32, n_head_kv=32, n_experts=None, n_experts_used=None, f_norm_eps=1e-05, rope_scaling_type=None, f_rope_freq_base=None, f_rope_scale=None, n_orig_ctx=None, rope_finetuned=None, ftype=None, path_model=PosixPath('models/7B-v2'))\n",
      "32000 32000\n",
      "Vocab info: <VocabLoader with 32000 base tokens and 0 added tokens>\n",
      "Special vocab info: <SpecialVocab with 61249 merges, special tokens {'bos': 1, 'eos': 2, 'unk': 0}, add special tokens {'bos': True, 'eos': False}>\n",
      "tok_embeddings.weight                            -> token_embd.weight                        | BF16   | [32000, 4096]\n",
      "norm.weight                                      -> output_norm.weight                       | BF16   | [4096]\n",
      "output.weight                                    -> output.weight                            | BF16   | [32000, 4096]\n",
      "layers.0.attention.wq.weight                     -> blk.0.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.0.attention.wk.weight                     -> blk.0.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.0.attention.wv.weight                     -> blk.0.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.0.attention.wo.weight                     -> blk.0.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.0.feed_forward.w1.weight                  -> blk.0.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.0.feed_forward.w2.weight                  -> blk.0.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.0.feed_forward.w3.weight                  -> blk.0.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.0.attention_norm.weight                   -> blk.0.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.0.ffn_norm.weight                         -> blk.0.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.1.attention.wq.weight                     -> blk.1.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.1.attention.wk.weight                     -> blk.1.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.1.attention.wv.weight                     -> blk.1.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.1.attention.wo.weight                     -> blk.1.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.1.feed_forward.w1.weight                  -> blk.1.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.1.feed_forward.w2.weight                  -> blk.1.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.1.feed_forward.w3.weight                  -> blk.1.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.1.attention_norm.weight                   -> blk.1.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.1.ffn_norm.weight                         -> blk.1.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.2.attention.wq.weight                     -> blk.2.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.2.attention.wk.weight                     -> blk.2.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.2.attention.wv.weight                     -> blk.2.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.2.attention.wo.weight                     -> blk.2.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.2.feed_forward.w1.weight                  -> blk.2.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.2.feed_forward.w2.weight                  -> blk.2.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.2.feed_forward.w3.weight                  -> blk.2.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.2.attention_norm.weight                   -> blk.2.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.2.ffn_norm.weight                         -> blk.2.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.3.attention.wq.weight                     -> blk.3.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.3.attention.wk.weight                     -> blk.3.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.3.attention.wv.weight                     -> blk.3.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.3.attention.wo.weight                     -> blk.3.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.3.feed_forward.w1.weight                  -> blk.3.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.3.feed_forward.w2.weight                  -> blk.3.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.3.feed_forward.w3.weight                  -> blk.3.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.3.attention_norm.weight                   -> blk.3.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.3.ffn_norm.weight                         -> blk.3.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.4.attention.wq.weight                     -> blk.4.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.4.attention.wk.weight                     -> blk.4.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.4.attention.wv.weight                     -> blk.4.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.4.attention.wo.weight                     -> blk.4.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.4.feed_forward.w1.weight                  -> blk.4.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.4.feed_forward.w2.weight                  -> blk.4.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.4.feed_forward.w3.weight                  -> blk.4.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.4.attention_norm.weight                   -> blk.4.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.4.ffn_norm.weight                         -> blk.4.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.5.attention.wq.weight                     -> blk.5.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.5.attention.wk.weight                     -> blk.5.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.5.attention.wv.weight                     -> blk.5.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.5.attention.wo.weight                     -> blk.5.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.5.feed_forward.w1.weight                  -> blk.5.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.5.feed_forward.w2.weight                  -> blk.5.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.5.feed_forward.w3.weight                  -> blk.5.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.5.attention_norm.weight                   -> blk.5.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.5.ffn_norm.weight                         -> blk.5.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.6.attention.wq.weight                     -> blk.6.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.6.attention.wk.weight                     -> blk.6.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.6.attention.wv.weight                     -> blk.6.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.6.attention.wo.weight                     -> blk.6.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.6.feed_forward.w1.weight                  -> blk.6.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.6.feed_forward.w2.weight                  -> blk.6.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.6.feed_forward.w3.weight                  -> blk.6.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.6.attention_norm.weight                   -> blk.6.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.6.ffn_norm.weight                         -> blk.6.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.7.attention.wq.weight                     -> blk.7.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.7.attention.wk.weight                     -> blk.7.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.7.attention.wv.weight                     -> blk.7.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.7.attention.wo.weight                     -> blk.7.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.7.feed_forward.w1.weight                  -> blk.7.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.7.feed_forward.w2.weight                  -> blk.7.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.7.feed_forward.w3.weight                  -> blk.7.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.7.attention_norm.weight                   -> blk.7.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.7.ffn_norm.weight                         -> blk.7.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.8.attention.wq.weight                     -> blk.8.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.8.attention.wk.weight                     -> blk.8.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.8.attention.wv.weight                     -> blk.8.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.8.attention.wo.weight                     -> blk.8.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.8.feed_forward.w1.weight                  -> blk.8.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.8.feed_forward.w2.weight                  -> blk.8.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.8.feed_forward.w3.weight                  -> blk.8.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.8.attention_norm.weight                   -> blk.8.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.8.ffn_norm.weight                         -> blk.8.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.9.attention.wq.weight                     -> blk.9.attn_q.weight                      | BF16   | [4096, 4096]\n",
      "layers.9.attention.wk.weight                     -> blk.9.attn_k.weight                      | BF16   | [4096, 4096]\n",
      "layers.9.attention.wv.weight                     -> blk.9.attn_v.weight                      | BF16   | [4096, 4096]\n",
      "layers.9.attention.wo.weight                     -> blk.9.attn_output.weight                 | BF16   | [4096, 4096]\n",
      "layers.9.feed_forward.w1.weight                  -> blk.9.ffn_gate.weight                    | BF16   | [11008, 4096]\n",
      "layers.9.feed_forward.w2.weight                  -> blk.9.ffn_down.weight                    | BF16   | [4096, 11008]\n",
      "layers.9.feed_forward.w3.weight                  -> blk.9.ffn_up.weight                      | BF16   | [11008, 4096]\n",
      "layers.9.attention_norm.weight                   -> blk.9.attn_norm.weight                   | BF16   | [4096]\n",
      "layers.9.ffn_norm.weight                         -> blk.9.ffn_norm.weight                    | BF16   | [4096]\n",
      "layers.10.attention.wq.weight                    -> blk.10.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.10.attention.wk.weight                    -> blk.10.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.10.attention.wv.weight                    -> blk.10.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.10.attention.wo.weight                    -> blk.10.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.10.feed_forward.w1.weight                 -> blk.10.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.10.feed_forward.w2.weight                 -> blk.10.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.10.feed_forward.w3.weight                 -> blk.10.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.10.attention_norm.weight                  -> blk.10.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.10.ffn_norm.weight                        -> blk.10.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.11.attention.wq.weight                    -> blk.11.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.11.attention.wk.weight                    -> blk.11.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.11.attention.wv.weight                    -> blk.11.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.11.attention.wo.weight                    -> blk.11.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.11.feed_forward.w1.weight                 -> blk.11.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.11.feed_forward.w2.weight                 -> blk.11.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.11.feed_forward.w3.weight                 -> blk.11.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.11.attention_norm.weight                  -> blk.11.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.11.ffn_norm.weight                        -> blk.11.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.12.attention.wq.weight                    -> blk.12.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.12.attention.wk.weight                    -> blk.12.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.12.attention.wv.weight                    -> blk.12.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.12.attention.wo.weight                    -> blk.12.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.12.feed_forward.w1.weight                 -> blk.12.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.12.feed_forward.w2.weight                 -> blk.12.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.12.feed_forward.w3.weight                 -> blk.12.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.12.attention_norm.weight                  -> blk.12.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.12.ffn_norm.weight                        -> blk.12.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.13.attention.wq.weight                    -> blk.13.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.13.attention.wk.weight                    -> blk.13.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.13.attention.wv.weight                    -> blk.13.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.13.attention.wo.weight                    -> blk.13.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.13.feed_forward.w1.weight                 -> blk.13.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.13.feed_forward.w2.weight                 -> blk.13.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.13.feed_forward.w3.weight                 -> blk.13.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.13.attention_norm.weight                  -> blk.13.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.13.ffn_norm.weight                        -> blk.13.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.14.attention.wq.weight                    -> blk.14.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.14.attention.wk.weight                    -> blk.14.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.14.attention.wv.weight                    -> blk.14.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.14.attention.wo.weight                    -> blk.14.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.14.feed_forward.w1.weight                 -> blk.14.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.14.feed_forward.w2.weight                 -> blk.14.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.14.feed_forward.w3.weight                 -> blk.14.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.14.attention_norm.weight                  -> blk.14.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.14.ffn_norm.weight                        -> blk.14.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.15.attention.wq.weight                    -> blk.15.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.15.attention.wk.weight                    -> blk.15.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.15.attention.wv.weight                    -> blk.15.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.15.attention.wo.weight                    -> blk.15.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.15.feed_forward.w1.weight                 -> blk.15.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.15.feed_forward.w2.weight                 -> blk.15.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.15.feed_forward.w3.weight                 -> blk.15.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.15.attention_norm.weight                  -> blk.15.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.15.ffn_norm.weight                        -> blk.15.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.16.attention.wq.weight                    -> blk.16.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.16.attention.wk.weight                    -> blk.16.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.16.attention.wv.weight                    -> blk.16.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.16.attention.wo.weight                    -> blk.16.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.16.feed_forward.w1.weight                 -> blk.16.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.16.feed_forward.w2.weight                 -> blk.16.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.16.feed_forward.w3.weight                 -> blk.16.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.16.attention_norm.weight                  -> blk.16.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.16.ffn_norm.weight                        -> blk.16.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.17.attention.wq.weight                    -> blk.17.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.17.attention.wk.weight                    -> blk.17.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.17.attention.wv.weight                    -> blk.17.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.17.attention.wo.weight                    -> blk.17.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.17.feed_forward.w1.weight                 -> blk.17.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.17.feed_forward.w2.weight                 -> blk.17.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.17.feed_forward.w3.weight                 -> blk.17.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.17.attention_norm.weight                  -> blk.17.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.17.ffn_norm.weight                        -> blk.17.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.18.attention.wq.weight                    -> blk.18.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.18.attention.wk.weight                    -> blk.18.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.18.attention.wv.weight                    -> blk.18.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.18.attention.wo.weight                    -> blk.18.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.18.feed_forward.w1.weight                 -> blk.18.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.18.feed_forward.w2.weight                 -> blk.18.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.18.feed_forward.w3.weight                 -> blk.18.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.18.attention_norm.weight                  -> blk.18.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.18.ffn_norm.weight                        -> blk.18.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.19.attention.wq.weight                    -> blk.19.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.19.attention.wk.weight                    -> blk.19.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.19.attention.wv.weight                    -> blk.19.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.19.attention.wo.weight                    -> blk.19.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.19.feed_forward.w1.weight                 -> blk.19.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.19.feed_forward.w2.weight                 -> blk.19.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.19.feed_forward.w3.weight                 -> blk.19.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.19.attention_norm.weight                  -> blk.19.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.19.ffn_norm.weight                        -> blk.19.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.20.attention.wq.weight                    -> blk.20.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.20.attention.wk.weight                    -> blk.20.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.20.attention.wv.weight                    -> blk.20.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.20.attention.wo.weight                    -> blk.20.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.20.feed_forward.w1.weight                 -> blk.20.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.20.feed_forward.w2.weight                 -> blk.20.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.20.feed_forward.w3.weight                 -> blk.20.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.20.attention_norm.weight                  -> blk.20.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.20.ffn_norm.weight                        -> blk.20.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.21.attention.wq.weight                    -> blk.21.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.21.attention.wk.weight                    -> blk.21.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.21.attention.wv.weight                    -> blk.21.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.21.attention.wo.weight                    -> blk.21.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.21.feed_forward.w1.weight                 -> blk.21.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.21.feed_forward.w2.weight                 -> blk.21.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.21.feed_forward.w3.weight                 -> blk.21.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.21.attention_norm.weight                  -> blk.21.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.21.ffn_norm.weight                        -> blk.21.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.22.attention.wq.weight                    -> blk.22.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.22.attention.wk.weight                    -> blk.22.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.22.attention.wv.weight                    -> blk.22.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.22.attention.wo.weight                    -> blk.22.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.22.feed_forward.w1.weight                 -> blk.22.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.22.feed_forward.w2.weight                 -> blk.22.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.22.feed_forward.w3.weight                 -> blk.22.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.22.attention_norm.weight                  -> blk.22.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.22.ffn_norm.weight                        -> blk.22.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.23.attention.wq.weight                    -> blk.23.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.23.attention.wk.weight                    -> blk.23.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.23.attention.wv.weight                    -> blk.23.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.23.attention.wo.weight                    -> blk.23.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.23.feed_forward.w1.weight                 -> blk.23.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.23.feed_forward.w2.weight                 -> blk.23.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.23.feed_forward.w3.weight                 -> blk.23.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.23.attention_norm.weight                  -> blk.23.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.23.ffn_norm.weight                        -> blk.23.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.24.attention.wq.weight                    -> blk.24.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.24.attention.wk.weight                    -> blk.24.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.24.attention.wv.weight                    -> blk.24.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.24.attention.wo.weight                    -> blk.24.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.24.feed_forward.w1.weight                 -> blk.24.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.24.feed_forward.w2.weight                 -> blk.24.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.24.feed_forward.w3.weight                 -> blk.24.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.24.attention_norm.weight                  -> blk.24.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.24.ffn_norm.weight                        -> blk.24.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.25.attention.wq.weight                    -> blk.25.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.25.attention.wk.weight                    -> blk.25.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.25.attention.wv.weight                    -> blk.25.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.25.attention.wo.weight                    -> blk.25.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.25.feed_forward.w1.weight                 -> blk.25.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.25.feed_forward.w2.weight                 -> blk.25.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.25.feed_forward.w3.weight                 -> blk.25.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.25.attention_norm.weight                  -> blk.25.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.25.ffn_norm.weight                        -> blk.25.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.26.attention.wq.weight                    -> blk.26.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.26.attention.wk.weight                    -> blk.26.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.26.attention.wv.weight                    -> blk.26.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.26.attention.wo.weight                    -> blk.26.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.26.feed_forward.w1.weight                 -> blk.26.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.26.feed_forward.w2.weight                 -> blk.26.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.26.feed_forward.w3.weight                 -> blk.26.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.26.attention_norm.weight                  -> blk.26.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.26.ffn_norm.weight                        -> blk.26.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.27.attention.wq.weight                    -> blk.27.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.27.attention.wk.weight                    -> blk.27.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.27.attention.wv.weight                    -> blk.27.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.27.attention.wo.weight                    -> blk.27.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.27.feed_forward.w1.weight                 -> blk.27.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.27.feed_forward.w2.weight                 -> blk.27.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.27.feed_forward.w3.weight                 -> blk.27.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.27.attention_norm.weight                  -> blk.27.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.27.ffn_norm.weight                        -> blk.27.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.28.attention.wq.weight                    -> blk.28.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.28.attention.wk.weight                    -> blk.28.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.28.attention.wv.weight                    -> blk.28.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.28.attention.wo.weight                    -> blk.28.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.28.feed_forward.w1.weight                 -> blk.28.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.28.feed_forward.w2.weight                 -> blk.28.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.28.feed_forward.w3.weight                 -> blk.28.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.28.attention_norm.weight                  -> blk.28.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.28.ffn_norm.weight                        -> blk.28.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.29.attention.wq.weight                    -> blk.29.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.29.attention.wk.weight                    -> blk.29.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.29.attention.wv.weight                    -> blk.29.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.29.attention.wo.weight                    -> blk.29.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.29.feed_forward.w1.weight                 -> blk.29.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.29.feed_forward.w2.weight                 -> blk.29.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.29.feed_forward.w3.weight                 -> blk.29.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.29.attention_norm.weight                  -> blk.29.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.29.ffn_norm.weight                        -> blk.29.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.30.attention.wq.weight                    -> blk.30.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.30.attention.wk.weight                    -> blk.30.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.30.attention.wv.weight                    -> blk.30.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.30.attention.wo.weight                    -> blk.30.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.30.feed_forward.w1.weight                 -> blk.30.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.30.feed_forward.w2.weight                 -> blk.30.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.30.feed_forward.w3.weight                 -> blk.30.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.30.attention_norm.weight                  -> blk.30.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.30.ffn_norm.weight                        -> blk.30.ffn_norm.weight                   | BF16   | [4096]\n",
      "layers.31.attention.wq.weight                    -> blk.31.attn_q.weight                     | BF16   | [4096, 4096]\n",
      "layers.31.attention.wk.weight                    -> blk.31.attn_k.weight                     | BF16   | [4096, 4096]\n",
      "layers.31.attention.wv.weight                    -> blk.31.attn_v.weight                     | BF16   | [4096, 4096]\n",
      "layers.31.attention.wo.weight                    -> blk.31.attn_output.weight                | BF16   | [4096, 4096]\n",
      "layers.31.feed_forward.w1.weight                 -> blk.31.ffn_gate.weight                   | BF16   | [11008, 4096]\n",
      "layers.31.feed_forward.w2.weight                 -> blk.31.ffn_down.weight                   | BF16   | [4096, 11008]\n",
      "layers.31.feed_forward.w3.weight                 -> blk.31.ffn_up.weight                     | BF16   | [11008, 4096]\n",
      "layers.31.attention_norm.weight                  -> blk.31.attn_norm.weight                  | BF16   | [4096]\n",
      "layers.31.ffn_norm.weight                        -> blk.31.ffn_norm.weight                   | BF16   | [4096]\n",
      "skipping tensor rope_freqs\n",
      "Writing models/7B-v2/ggml-model-f16.gguf, format 1\n",
      "gguf: This GGUF file is for Little Endian only\n",
      "gguf: Adding 61249 merge(s).\n",
      "gguf: Setting special token type bos to 1\n",
      "gguf: Setting special token type eos to 2\n",
      "gguf: Setting special token type unk to 0\n",
      "gguf: Setting add_bos_token to True\n",
      "gguf: Setting add_eos_token to False\n",
      "[  1/291] Writing tensor token_embd.weight                      | size  32000 x   4096  | type F16  | T+   1\n",
      "[  2/291] Writing tensor output_norm.weight                     | size   4096           | type F32  | T+   1\n",
      "[  3/291] Writing tensor output.weight                          | size  32000 x   4096  | type F16  | T+   1\n",
      "[  4/291] Writing tensor blk.0.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
      "[  5/291] Writing tensor blk.0.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
      "[  6/291] Writing tensor blk.0.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
      "[  7/291] Writing tensor blk.0.attn_output.weight               | size   4096 x   4096  | type F16  | T+   1\n",
      "[  8/291] Writing tensor blk.0.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   1\n",
      "[  9/291] Writing tensor blk.0.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   1\n",
      "[ 10/291] Writing tensor blk.0.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   1\n",
      "[ 11/291] Writing tensor blk.0.attn_norm.weight                 | size   4096           | type F32  | T+   1\n",
      "[ 12/291] Writing tensor blk.0.ffn_norm.weight                  | size   4096           | type F32  | T+   1\n",
      "[ 13/291] Writing tensor blk.1.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
      "[ 14/291] Writing tensor blk.1.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
      "[ 15/291] Writing tensor blk.1.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   1\n",
      "[ 16/291] Writing tensor blk.1.attn_output.weight               | size   4096 x   4096  | type F16  | T+   1\n",
      "[ 17/291] Writing tensor blk.1.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   1\n",
      "[ 18/291] Writing tensor blk.1.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   1\n",
      "[ 19/291] Writing tensor blk.1.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   1\n",
      "[ 20/291] Writing tensor blk.1.attn_norm.weight                 | size   4096           | type F32  | T+   2\n",
      "[ 21/291] Writing tensor blk.1.ffn_norm.weight                  | size   4096           | type F32  | T+   2\n",
      "[ 22/291] Writing tensor blk.2.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 23/291] Writing tensor blk.2.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 24/291] Writing tensor blk.2.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 25/291] Writing tensor blk.2.attn_output.weight               | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 26/291] Writing tensor blk.2.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   2\n",
      "[ 27/291] Writing tensor blk.2.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   2\n",
      "[ 28/291] Writing tensor blk.2.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   2\n",
      "[ 29/291] Writing tensor blk.2.attn_norm.weight                 | size   4096           | type F32  | T+   2\n",
      "[ 30/291] Writing tensor blk.2.ffn_norm.weight                  | size   4096           | type F32  | T+   2\n",
      "[ 31/291] Writing tensor blk.3.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 32/291] Writing tensor blk.3.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 33/291] Writing tensor blk.3.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 34/291] Writing tensor blk.3.attn_output.weight               | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 35/291] Writing tensor blk.3.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   2\n",
      "[ 36/291] Writing tensor blk.3.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   2\n",
      "[ 37/291] Writing tensor blk.3.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   2\n",
      "[ 38/291] Writing tensor blk.3.attn_norm.weight                 | size   4096           | type F32  | T+   2\n",
      "[ 39/291] Writing tensor blk.3.ffn_norm.weight                  | size   4096           | type F32  | T+   2\n",
      "[ 40/291] Writing tensor blk.4.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 41/291] Writing tensor blk.4.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 42/291] Writing tensor blk.4.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 43/291] Writing tensor blk.4.attn_output.weight               | size   4096 x   4096  | type F16  | T+   2\n",
      "[ 44/291] Writing tensor blk.4.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   3\n",
      "[ 45/291] Writing tensor blk.4.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   3\n",
      "[ 46/291] Writing tensor blk.4.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   3\n",
      "[ 47/291] Writing tensor blk.4.attn_norm.weight                 | size   4096           | type F32  | T+   3\n",
      "[ 48/291] Writing tensor blk.4.ffn_norm.weight                  | size   4096           | type F32  | T+   3\n",
      "[ 49/291] Writing tensor blk.5.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
      "[ 50/291] Writing tensor blk.5.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
      "[ 51/291] Writing tensor blk.5.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
      "[ 52/291] Writing tensor blk.5.attn_output.weight               | size   4096 x   4096  | type F16  | T+   3\n",
      "[ 53/291] Writing tensor blk.5.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   3\n",
      "[ 54/291] Writing tensor blk.5.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   3\n",
      "[ 55/291] Writing tensor blk.5.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   3\n",
      "[ 56/291] Writing tensor blk.5.attn_norm.weight                 | size   4096           | type F32  | T+   3\n",
      "[ 57/291] Writing tensor blk.5.ffn_norm.weight                  | size   4096           | type F32  | T+   3\n",
      "[ 58/291] Writing tensor blk.6.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
      "[ 59/291] Writing tensor blk.6.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
      "[ 60/291] Writing tensor blk.6.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   3\n",
      "[ 61/291] Writing tensor blk.6.attn_output.weight               | size   4096 x   4096  | type F16  | T+   3\n",
      "[ 62/291] Writing tensor blk.6.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   3\n",
      "[ 63/291] Writing tensor blk.6.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   3\n",
      "[ 64/291] Writing tensor blk.6.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   4\n",
      "[ 65/291] Writing tensor blk.6.attn_norm.weight                 | size   4096           | type F32  | T+   4\n",
      "[ 66/291] Writing tensor blk.6.ffn_norm.weight                  | size   4096           | type F32  | T+   4\n",
      "[ 67/291] Writing tensor blk.7.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 68/291] Writing tensor blk.7.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 69/291] Writing tensor blk.7.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 70/291] Writing tensor blk.7.attn_output.weight               | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 71/291] Writing tensor blk.7.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   4\n",
      "[ 72/291] Writing tensor blk.7.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   4\n",
      "[ 73/291] Writing tensor blk.7.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   4\n",
      "[ 74/291] Writing tensor blk.7.attn_norm.weight                 | size   4096           | type F32  | T+   4\n",
      "[ 75/291] Writing tensor blk.7.ffn_norm.weight                  | size   4096           | type F32  | T+   4\n",
      "[ 76/291] Writing tensor blk.8.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 77/291] Writing tensor blk.8.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 78/291] Writing tensor blk.8.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 79/291] Writing tensor blk.8.attn_output.weight               | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 80/291] Writing tensor blk.8.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   4\n",
      "[ 81/291] Writing tensor blk.8.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   4\n",
      "[ 82/291] Writing tensor blk.8.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   4\n",
      "[ 83/291] Writing tensor blk.8.attn_norm.weight                 | size   4096           | type F32  | T+   4\n",
      "[ 84/291] Writing tensor blk.8.ffn_norm.weight                  | size   4096           | type F32  | T+   4\n",
      "[ 85/291] Writing tensor blk.9.attn_q.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 86/291] Writing tensor blk.9.attn_k.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 87/291] Writing tensor blk.9.attn_v.weight                    | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 88/291] Writing tensor blk.9.attn_output.weight               | size   4096 x   4096  | type F16  | T+   4\n",
      "[ 89/291] Writing tensor blk.9.ffn_gate.weight                  | size  11008 x   4096  | type F16  | T+   5\n",
      "[ 90/291] Writing tensor blk.9.ffn_down.weight                  | size   4096 x  11008  | type F16  | T+   5\n",
      "[ 91/291] Writing tensor blk.9.ffn_up.weight                    | size  11008 x   4096  | type F16  | T+   5\n",
      "[ 92/291] Writing tensor blk.9.attn_norm.weight                 | size   4096           | type F32  | T+   5\n",
      "[ 93/291] Writing tensor blk.9.ffn_norm.weight                  | size   4096           | type F32  | T+   5\n",
      "[ 94/291] Writing tensor blk.10.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   5\n",
      "[ 95/291] Writing tensor blk.10.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   5\n",
      "[ 96/291] Writing tensor blk.10.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   5\n",
      "[ 97/291] Writing tensor blk.10.attn_output.weight              | size   4096 x   4096  | type F16  | T+   5\n",
      "[ 98/291] Writing tensor blk.10.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   5\n",
      "[ 99/291] Writing tensor blk.10.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   5\n",
      "[100/291] Writing tensor blk.10.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   5\n",
      "[101/291] Writing tensor blk.10.attn_norm.weight                | size   4096           | type F32  | T+   5\n",
      "[102/291] Writing tensor blk.10.ffn_norm.weight                 | size   4096           | type F32  | T+   5\n",
      "[103/291] Writing tensor blk.11.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   5\n",
      "[104/291] Writing tensor blk.11.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   5\n",
      "[105/291] Writing tensor blk.11.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   5\n",
      "[106/291] Writing tensor blk.11.attn_output.weight              | size   4096 x   4096  | type F16  | T+   5\n",
      "[107/291] Writing tensor blk.11.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   5\n",
      "[108/291] Writing tensor blk.11.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   6\n",
      "[109/291] Writing tensor blk.11.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   6\n",
      "[110/291] Writing tensor blk.11.attn_norm.weight                | size   4096           | type F32  | T+   6\n",
      "[111/291] Writing tensor blk.11.ffn_norm.weight                 | size   4096           | type F32  | T+   6\n",
      "[112/291] Writing tensor blk.12.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
      "[113/291] Writing tensor blk.12.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
      "[114/291] Writing tensor blk.12.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
      "[115/291] Writing tensor blk.12.attn_output.weight              | size   4096 x   4096  | type F16  | T+   6\n",
      "[116/291] Writing tensor blk.12.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   6\n",
      "[117/291] Writing tensor blk.12.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   6\n",
      "[118/291] Writing tensor blk.12.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   6\n",
      "[119/291] Writing tensor blk.12.attn_norm.weight                | size   4096           | type F32  | T+   6\n",
      "[120/291] Writing tensor blk.12.ffn_norm.weight                 | size   4096           | type F32  | T+   6\n",
      "[121/291] Writing tensor blk.13.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
      "[122/291] Writing tensor blk.13.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
      "[123/291] Writing tensor blk.13.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
      "[124/291] Writing tensor blk.13.attn_output.weight              | size   4096 x   4096  | type F16  | T+   6\n",
      "[125/291] Writing tensor blk.13.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   6\n",
      "[126/291] Writing tensor blk.13.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   6\n",
      "[127/291] Writing tensor blk.13.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   6\n",
      "[128/291] Writing tensor blk.13.attn_norm.weight                | size   4096           | type F32  | T+   6\n",
      "[129/291] Writing tensor blk.13.ffn_norm.weight                 | size   4096           | type F32  | T+   6\n",
      "[130/291] Writing tensor blk.14.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
      "[131/291] Writing tensor blk.14.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
      "[132/291] Writing tensor blk.14.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   6\n",
      "[133/291] Writing tensor blk.14.attn_output.weight              | size   4096 x   4096  | type F16  | T+   6\n",
      "[134/291] Writing tensor blk.14.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   7\n",
      "[135/291] Writing tensor blk.14.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   7\n",
      "[136/291] Writing tensor blk.14.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   7\n",
      "[137/291] Writing tensor blk.14.attn_norm.weight                | size   4096           | type F32  | T+   7\n",
      "[138/291] Writing tensor blk.14.ffn_norm.weight                 | size   4096           | type F32  | T+   7\n",
      "[139/291] Writing tensor blk.15.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   7\n",
      "[140/291] Writing tensor blk.15.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   7\n",
      "[141/291] Writing tensor blk.15.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   7\n",
      "[142/291] Writing tensor blk.15.attn_output.weight              | size   4096 x   4096  | type F16  | T+   7\n",
      "[143/291] Writing tensor blk.15.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   7\n",
      "[144/291] Writing tensor blk.15.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   7\n",
      "[145/291] Writing tensor blk.15.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   7\n",
      "[146/291] Writing tensor blk.15.attn_norm.weight                | size   4096           | type F32  | T+   7\n",
      "[147/291] Writing tensor blk.15.ffn_norm.weight                 | size   4096           | type F32  | T+   7\n",
      "[148/291] Writing tensor blk.16.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   7\n",
      "[149/291] Writing tensor blk.16.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   7\n",
      "[150/291] Writing tensor blk.16.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   7\n",
      "[151/291] Writing tensor blk.16.attn_output.weight              | size   4096 x   4096  | type F16  | T+   7\n",
      "[152/291] Writing tensor blk.16.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   7\n",
      "[153/291] Writing tensor blk.16.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   8\n",
      "[154/291] Writing tensor blk.16.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   8\n",
      "[155/291] Writing tensor blk.16.attn_norm.weight                | size   4096           | type F32  | T+   8\n",
      "[156/291] Writing tensor blk.16.ffn_norm.weight                 | size   4096           | type F32  | T+   8\n",
      "[157/291] Writing tensor blk.17.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
      "[158/291] Writing tensor blk.17.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
      "[159/291] Writing tensor blk.17.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
      "[160/291] Writing tensor blk.17.attn_output.weight              | size   4096 x   4096  | type F16  | T+   8\n",
      "[161/291] Writing tensor blk.17.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   8\n",
      "[162/291] Writing tensor blk.17.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   8\n",
      "[163/291] Writing tensor blk.17.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   8\n",
      "[164/291] Writing tensor blk.17.attn_norm.weight                | size   4096           | type F32  | T+   8\n",
      "[165/291] Writing tensor blk.17.ffn_norm.weight                 | size   4096           | type F32  | T+   8\n",
      "[166/291] Writing tensor blk.18.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
      "[167/291] Writing tensor blk.18.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
      "[168/291] Writing tensor blk.18.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
      "[169/291] Writing tensor blk.18.attn_output.weight              | size   4096 x   4096  | type F16  | T+   8\n",
      "[170/291] Writing tensor blk.18.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   8\n",
      "[171/291] Writing tensor blk.18.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   8\n",
      "[172/291] Writing tensor blk.18.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   8\n",
      "[173/291] Writing tensor blk.18.attn_norm.weight                | size   4096           | type F32  | T+   8\n",
      "[174/291] Writing tensor blk.18.ffn_norm.weight                 | size   4096           | type F32  | T+   8\n",
      "[175/291] Writing tensor blk.19.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
      "[176/291] Writing tensor blk.19.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
      "[177/291] Writing tensor blk.19.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   8\n",
      "[178/291] Writing tensor blk.19.attn_output.weight              | size   4096 x   4096  | type F16  | T+   8\n",
      "[179/291] Writing tensor blk.19.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   9\n",
      "[180/291] Writing tensor blk.19.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   9\n",
      "[181/291] Writing tensor blk.19.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   9\n",
      "[182/291] Writing tensor blk.19.attn_norm.weight                | size   4096           | type F32  | T+   9\n",
      "[183/291] Writing tensor blk.19.ffn_norm.weight                 | size   4096           | type F32  | T+   9\n",
      "[184/291] Writing tensor blk.20.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   9\n",
      "[185/291] Writing tensor blk.20.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   9\n",
      "[186/291] Writing tensor blk.20.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   9\n",
      "[187/291] Writing tensor blk.20.attn_output.weight              | size   4096 x   4096  | type F16  | T+   9\n",
      "[188/291] Writing tensor blk.20.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   9\n",
      "[189/291] Writing tensor blk.20.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+   9\n",
      "[190/291] Writing tensor blk.20.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+   9\n",
      "[191/291] Writing tensor blk.20.attn_norm.weight                | size   4096           | type F32  | T+   9\n",
      "[192/291] Writing tensor blk.20.ffn_norm.weight                 | size   4096           | type F32  | T+   9\n",
      "[193/291] Writing tensor blk.21.attn_q.weight                   | size   4096 x   4096  | type F16  | T+   9\n",
      "[194/291] Writing tensor blk.21.attn_k.weight                   | size   4096 x   4096  | type F16  | T+   9\n",
      "[195/291] Writing tensor blk.21.attn_v.weight                   | size   4096 x   4096  | type F16  | T+   9\n",
      "[196/291] Writing tensor blk.21.attn_output.weight              | size   4096 x   4096  | type F16  | T+   9\n",
      "[197/291] Writing tensor blk.21.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+   9\n",
      "[198/291] Writing tensor blk.21.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  10\n",
      "[199/291] Writing tensor blk.21.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  10\n",
      "[200/291] Writing tensor blk.21.attn_norm.weight                | size   4096           | type F32  | T+  10\n",
      "[201/291] Writing tensor blk.21.ffn_norm.weight                 | size   4096           | type F32  | T+  10\n",
      "[202/291] Writing tensor blk.22.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
      "[203/291] Writing tensor blk.22.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
      "[204/291] Writing tensor blk.22.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
      "[205/291] Writing tensor blk.22.attn_output.weight              | size   4096 x   4096  | type F16  | T+  10\n",
      "[206/291] Writing tensor blk.22.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  10\n",
      "[207/291] Writing tensor blk.22.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  10\n",
      "[208/291] Writing tensor blk.22.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  10\n",
      "[209/291] Writing tensor blk.22.attn_norm.weight                | size   4096           | type F32  | T+  10\n",
      "[210/291] Writing tensor blk.22.ffn_norm.weight                 | size   4096           | type F32  | T+  10\n",
      "[211/291] Writing tensor blk.23.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
      "[212/291] Writing tensor blk.23.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
      "[213/291] Writing tensor blk.23.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
      "[214/291] Writing tensor blk.23.attn_output.weight              | size   4096 x   4096  | type F16  | T+  10\n",
      "[215/291] Writing tensor blk.23.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  10\n",
      "[216/291] Writing tensor blk.23.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  10\n",
      "[217/291] Writing tensor blk.23.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  10\n",
      "[218/291] Writing tensor blk.23.attn_norm.weight                | size   4096           | type F32  | T+  10\n",
      "[219/291] Writing tensor blk.23.ffn_norm.weight                 | size   4096           | type F32  | T+  10\n",
      "[220/291] Writing tensor blk.24.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
      "[221/291] Writing tensor blk.24.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  10\n",
      "[222/291] Writing tensor blk.24.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  11\n",
      "[223/291] Writing tensor blk.24.attn_output.weight              | size   4096 x   4096  | type F16  | T+  11\n",
      "[224/291] Writing tensor blk.24.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  11\n",
      "[225/291] Writing tensor blk.24.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  11\n",
      "[226/291] Writing tensor blk.24.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  11\n",
      "[227/291] Writing tensor blk.24.attn_norm.weight                | size   4096           | type F32  | T+  11\n",
      "[228/291] Writing tensor blk.24.ffn_norm.weight                 | size   4096           | type F32  | T+  11\n",
      "[229/291] Writing tensor blk.25.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  11\n",
      "[230/291] Writing tensor blk.25.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  11\n",
      "[231/291] Writing tensor blk.25.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  11\n",
      "[232/291] Writing tensor blk.25.attn_output.weight              | size   4096 x   4096  | type F16  | T+  11\n",
      "[233/291] Writing tensor blk.25.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  11\n",
      "[234/291] Writing tensor blk.25.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  11\n",
      "[235/291] Writing tensor blk.25.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  11\n",
      "[236/291] Writing tensor blk.25.attn_norm.weight                | size   4096           | type F32  | T+  11\n",
      "[237/291] Writing tensor blk.25.ffn_norm.weight                 | size   4096           | type F32  | T+  11\n",
      "[238/291] Writing tensor blk.26.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  11\n",
      "[239/291] Writing tensor blk.26.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  11\n",
      "[240/291] Writing tensor blk.26.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  11\n",
      "[241/291] Writing tensor blk.26.attn_output.weight              | size   4096 x   4096  | type F16  | T+  11\n",
      "[242/291] Writing tensor blk.26.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  12\n",
      "[243/291] Writing tensor blk.26.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  12\n",
      "[244/291] Writing tensor blk.26.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  12\n",
      "[245/291] Writing tensor blk.26.attn_norm.weight                | size   4096           | type F32  | T+  12\n",
      "[246/291] Writing tensor blk.26.ffn_norm.weight                 | size   4096           | type F32  | T+  12\n",
      "[247/291] Writing tensor blk.27.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  12\n",
      "[248/291] Writing tensor blk.27.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  12\n",
      "[249/291] Writing tensor blk.27.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  12\n",
      "[250/291] Writing tensor blk.27.attn_output.weight              | size   4096 x   4096  | type F16  | T+  12\n",
      "[251/291] Writing tensor blk.27.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  12\n",
      "[252/291] Writing tensor blk.27.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  12\n",
      "[253/291] Writing tensor blk.27.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  12\n",
      "[254/291] Writing tensor blk.27.attn_norm.weight                | size   4096           | type F32  | T+  12\n",
      "[255/291] Writing tensor blk.27.ffn_norm.weight                 | size   4096           | type F32  | T+  12\n",
      "[256/291] Writing tensor blk.28.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  12\n",
      "[257/291] Writing tensor blk.28.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  12\n",
      "[258/291] Writing tensor blk.28.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  12\n",
      "[259/291] Writing tensor blk.28.attn_output.weight              | size   4096 x   4096  | type F16  | T+  12\n",
      "[260/291] Writing tensor blk.28.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  12\n",
      "[261/291] Writing tensor blk.28.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  12\n",
      "[262/291] Writing tensor blk.28.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  12\n",
      "[263/291] Writing tensor blk.28.attn_norm.weight                | size   4096           | type F32  | T+  13\n",
      "[264/291] Writing tensor blk.28.ffn_norm.weight                 | size   4096           | type F32  | T+  13\n",
      "[265/291] Writing tensor blk.29.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
      "[266/291] Writing tensor blk.29.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
      "[267/291] Writing tensor blk.29.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
      "[268/291] Writing tensor blk.29.attn_output.weight              | size   4096 x   4096  | type F16  | T+  13\n",
      "[269/291] Writing tensor blk.29.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  13\n",
      "[270/291] Writing tensor blk.29.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  13\n",
      "[271/291] Writing tensor blk.29.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  13\n",
      "[272/291] Writing tensor blk.29.attn_norm.weight                | size   4096           | type F32  | T+  13\n",
      "[273/291] Writing tensor blk.29.ffn_norm.weight                 | size   4096           | type F32  | T+  13\n",
      "[274/291] Writing tensor blk.30.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
      "[275/291] Writing tensor blk.30.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
      "[276/291] Writing tensor blk.30.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
      "[277/291] Writing tensor blk.30.attn_output.weight              | size   4096 x   4096  | type F16  | T+  13\n",
      "[278/291] Writing tensor blk.30.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  13\n",
      "[279/291] Writing tensor blk.30.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  13\n",
      "[280/291] Writing tensor blk.30.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  13\n",
      "[281/291] Writing tensor blk.30.attn_norm.weight                | size   4096           | type F32  | T+  13\n",
      "[282/291] Writing tensor blk.30.ffn_norm.weight                 | size   4096           | type F32  | T+  13\n",
      "[283/291] Writing tensor blk.31.attn_q.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
      "[284/291] Writing tensor blk.31.attn_k.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
      "[285/291] Writing tensor blk.31.attn_v.weight                   | size   4096 x   4096  | type F16  | T+  13\n",
      "[286/291] Writing tensor blk.31.attn_output.weight              | size   4096 x   4096  | type F16  | T+  13\n",
      "[287/291] Writing tensor blk.31.ffn_gate.weight                 | size  11008 x   4096  | type F16  | T+  14\n",
      "[288/291] Writing tensor blk.31.ffn_down.weight                 | size   4096 x  11008  | type F16  | T+  14\n",
      "[289/291] Writing tensor blk.31.ffn_up.weight                   | size  11008 x   4096  | type F16  | T+  14\n",
      "[290/291] Writing tensor blk.31.attn_norm.weight                | size   4096           | type F32  | T+  14\n",
      "[291/291] Writing tensor blk.31.ffn_norm.weight                 | size   4096           | type F32  | T+  14\n",
      "Wrote models/7B-v2/ggml-model-f16.gguf\n",
      "Loading model file models/13B-v2/consolidated.00.pth\n",
      "Loading model file models/13B-v2/consolidated.01.pth\n",
      "params = Params(n_vocab=32000, n_embd=5120, n_layer=40, n_ctx=4096, n_ff=13824, n_head=40, n_head_kv=40, n_experts=None, n_experts_used=None, f_norm_eps=1e-05, rope_scaling_type=None, f_rope_freq_base=None, f_rope_scale=None, n_orig_ctx=None, rope_finetuned=None, ftype=None, path_model=PosixPath('models/13B-v2'))\n",
      "32000 32000\n",
      "Vocab info: <VocabLoader with 32000 base tokens and 0 added tokens>\n",
      "Special vocab info: <SpecialVocab with 61249 merges, special tokens {'bos': 1, 'eos': 2, 'unk': 0}, add special tokens {'bos': True, 'eos': False}>\n",
      "tok_embeddings.weight                            -> token_embd.weight                        | BF16   | [32000, 5120]\n",
      "norm.weight                                      -> output_norm.weight                       | BF16   | [5120]\n",
      "output.weight                                    -> output.weight                            | BF16   | [32000, 5120]\n",
      "layers.0.attention.wq.weight                     -> blk.0.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.0.attention.wk.weight                     -> blk.0.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.0.attention.wv.weight                     -> blk.0.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.0.attention.wo.weight                     -> blk.0.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.0.feed_forward.w1.weight                  -> blk.0.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.0.feed_forward.w2.weight                  -> blk.0.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.0.feed_forward.w3.weight                  -> blk.0.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.0.attention_norm.weight                   -> blk.0.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.0.ffn_norm.weight                         -> blk.0.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.1.attention.wq.weight                     -> blk.1.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.1.attention.wk.weight                     -> blk.1.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.1.attention.wv.weight                     -> blk.1.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.1.attention.wo.weight                     -> blk.1.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.1.feed_forward.w1.weight                  -> blk.1.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.1.feed_forward.w2.weight                  -> blk.1.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.1.feed_forward.w3.weight                  -> blk.1.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.1.attention_norm.weight                   -> blk.1.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.1.ffn_norm.weight                         -> blk.1.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.2.attention.wq.weight                     -> blk.2.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.2.attention.wk.weight                     -> blk.2.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.2.attention.wv.weight                     -> blk.2.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.2.attention.wo.weight                     -> blk.2.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.2.feed_forward.w1.weight                  -> blk.2.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.2.feed_forward.w2.weight                  -> blk.2.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.2.feed_forward.w3.weight                  -> blk.2.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.2.attention_norm.weight                   -> blk.2.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.2.ffn_norm.weight                         -> blk.2.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.3.attention.wq.weight                     -> blk.3.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.3.attention.wk.weight                     -> blk.3.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.3.attention.wv.weight                     -> blk.3.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.3.attention.wo.weight                     -> blk.3.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.3.feed_forward.w1.weight                  -> blk.3.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.3.feed_forward.w2.weight                  -> blk.3.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.3.feed_forward.w3.weight                  -> blk.3.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.3.attention_norm.weight                   -> blk.3.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.3.ffn_norm.weight                         -> blk.3.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.4.attention.wq.weight                     -> blk.4.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.4.attention.wk.weight                     -> blk.4.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.4.attention.wv.weight                     -> blk.4.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.4.attention.wo.weight                     -> blk.4.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.4.feed_forward.w1.weight                  -> blk.4.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.4.feed_forward.w2.weight                  -> blk.4.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.4.feed_forward.w3.weight                  -> blk.4.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.4.attention_norm.weight                   -> blk.4.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.4.ffn_norm.weight                         -> blk.4.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.5.attention.wq.weight                     -> blk.5.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.5.attention.wk.weight                     -> blk.5.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.5.attention.wv.weight                     -> blk.5.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.5.attention.wo.weight                     -> blk.5.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.5.feed_forward.w1.weight                  -> blk.5.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.5.feed_forward.w2.weight                  -> blk.5.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.5.feed_forward.w3.weight                  -> blk.5.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.5.attention_norm.weight                   -> blk.5.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.5.ffn_norm.weight                         -> blk.5.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.6.attention.wq.weight                     -> blk.6.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.6.attention.wk.weight                     -> blk.6.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.6.attention.wv.weight                     -> blk.6.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.6.attention.wo.weight                     -> blk.6.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.6.feed_forward.w1.weight                  -> blk.6.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.6.feed_forward.w2.weight                  -> blk.6.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.6.feed_forward.w3.weight                  -> blk.6.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.6.attention_norm.weight                   -> blk.6.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.6.ffn_norm.weight                         -> blk.6.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.7.attention.wq.weight                     -> blk.7.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.7.attention.wk.weight                     -> blk.7.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.7.attention.wv.weight                     -> blk.7.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.7.attention.wo.weight                     -> blk.7.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.7.feed_forward.w1.weight                  -> blk.7.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.7.feed_forward.w2.weight                  -> blk.7.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.7.feed_forward.w3.weight                  -> blk.7.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.7.attention_norm.weight                   -> blk.7.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.7.ffn_norm.weight                         -> blk.7.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.8.attention.wq.weight                     -> blk.8.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.8.attention.wk.weight                     -> blk.8.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.8.attention.wv.weight                     -> blk.8.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.8.attention.wo.weight                     -> blk.8.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.8.feed_forward.w1.weight                  -> blk.8.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.8.feed_forward.w2.weight                  -> blk.8.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.8.feed_forward.w3.weight                  -> blk.8.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.8.attention_norm.weight                   -> blk.8.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.8.ffn_norm.weight                         -> blk.8.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.9.attention.wq.weight                     -> blk.9.attn_q.weight                      | BF16   | [5120, 5120]\n",
      "layers.9.attention.wk.weight                     -> blk.9.attn_k.weight                      | BF16   | [5120, 5120]\n",
      "layers.9.attention.wv.weight                     -> blk.9.attn_v.weight                      | BF16   | [5120, 5120]\n",
      "layers.9.attention.wo.weight                     -> blk.9.attn_output.weight                 | BF16   | [5120, 5120]\n",
      "layers.9.feed_forward.w1.weight                  -> blk.9.ffn_gate.weight                    | BF16   | [13824, 5120]\n",
      "layers.9.feed_forward.w2.weight                  -> blk.9.ffn_down.weight                    | BF16   | [5120, 13824]\n",
      "layers.9.feed_forward.w3.weight                  -> blk.9.ffn_up.weight                      | BF16   | [13824, 5120]\n",
      "layers.9.attention_norm.weight                   -> blk.9.attn_norm.weight                   | BF16   | [5120]\n",
      "layers.9.ffn_norm.weight                         -> blk.9.ffn_norm.weight                    | BF16   | [5120]\n",
      "layers.10.attention.wq.weight                    -> blk.10.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.10.attention.wk.weight                    -> blk.10.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.10.attention.wv.weight                    -> blk.10.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.10.attention.wo.weight                    -> blk.10.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.10.feed_forward.w1.weight                 -> blk.10.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.10.feed_forward.w2.weight                 -> blk.10.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.10.feed_forward.w3.weight                 -> blk.10.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.10.attention_norm.weight                  -> blk.10.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.10.ffn_norm.weight                        -> blk.10.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.11.attention.wq.weight                    -> blk.11.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.11.attention.wk.weight                    -> blk.11.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.11.attention.wv.weight                    -> blk.11.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.11.attention.wo.weight                    -> blk.11.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.11.feed_forward.w1.weight                 -> blk.11.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.11.feed_forward.w2.weight                 -> blk.11.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.11.feed_forward.w3.weight                 -> blk.11.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.11.attention_norm.weight                  -> blk.11.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.11.ffn_norm.weight                        -> blk.11.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.12.attention.wq.weight                    -> blk.12.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.12.attention.wk.weight                    -> blk.12.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.12.attention.wv.weight                    -> blk.12.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.12.attention.wo.weight                    -> blk.12.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.12.feed_forward.w1.weight                 -> blk.12.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.12.feed_forward.w2.weight                 -> blk.12.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.12.feed_forward.w3.weight                 -> blk.12.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.12.attention_norm.weight                  -> blk.12.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.12.ffn_norm.weight                        -> blk.12.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.13.attention.wq.weight                    -> blk.13.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.13.attention.wk.weight                    -> blk.13.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.13.attention.wv.weight                    -> blk.13.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.13.attention.wo.weight                    -> blk.13.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.13.feed_forward.w1.weight                 -> blk.13.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.13.feed_forward.w2.weight                 -> blk.13.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.13.feed_forward.w3.weight                 -> blk.13.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.13.attention_norm.weight                  -> blk.13.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.13.ffn_norm.weight                        -> blk.13.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.14.attention.wq.weight                    -> blk.14.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.14.attention.wk.weight                    -> blk.14.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.14.attention.wv.weight                    -> blk.14.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.14.attention.wo.weight                    -> blk.14.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.14.feed_forward.w1.weight                 -> blk.14.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.14.feed_forward.w2.weight                 -> blk.14.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.14.feed_forward.w3.weight                 -> blk.14.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.14.attention_norm.weight                  -> blk.14.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.14.ffn_norm.weight                        -> blk.14.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.15.attention.wq.weight                    -> blk.15.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.15.attention.wk.weight                    -> blk.15.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.15.attention.wv.weight                    -> blk.15.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.15.attention.wo.weight                    -> blk.15.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.15.feed_forward.w1.weight                 -> blk.15.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.15.feed_forward.w2.weight                 -> blk.15.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.15.feed_forward.w3.weight                 -> blk.15.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.15.attention_norm.weight                  -> blk.15.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.15.ffn_norm.weight                        -> blk.15.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.16.attention.wq.weight                    -> blk.16.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.16.attention.wk.weight                    -> blk.16.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.16.attention.wv.weight                    -> blk.16.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.16.attention.wo.weight                    -> blk.16.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.16.feed_forward.w1.weight                 -> blk.16.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.16.feed_forward.w2.weight                 -> blk.16.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.16.feed_forward.w3.weight                 -> blk.16.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.16.attention_norm.weight                  -> blk.16.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.16.ffn_norm.weight                        -> blk.16.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.17.attention.wq.weight                    -> blk.17.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.17.attention.wk.weight                    -> blk.17.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.17.attention.wv.weight                    -> blk.17.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.17.attention.wo.weight                    -> blk.17.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.17.feed_forward.w1.weight                 -> blk.17.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.17.feed_forward.w2.weight                 -> blk.17.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.17.feed_forward.w3.weight                 -> blk.17.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.17.attention_norm.weight                  -> blk.17.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.17.ffn_norm.weight                        -> blk.17.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.18.attention.wq.weight                    -> blk.18.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.18.attention.wk.weight                    -> blk.18.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.18.attention.wv.weight                    -> blk.18.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.18.attention.wo.weight                    -> blk.18.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.18.feed_forward.w1.weight                 -> blk.18.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.18.feed_forward.w2.weight                 -> blk.18.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.18.feed_forward.w3.weight                 -> blk.18.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.18.attention_norm.weight                  -> blk.18.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.18.ffn_norm.weight                        -> blk.18.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.19.attention.wq.weight                    -> blk.19.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.19.attention.wk.weight                    -> blk.19.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.19.attention.wv.weight                    -> blk.19.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.19.attention.wo.weight                    -> blk.19.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.19.feed_forward.w1.weight                 -> blk.19.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.19.feed_forward.w2.weight                 -> blk.19.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.19.feed_forward.w3.weight                 -> blk.19.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.19.attention_norm.weight                  -> blk.19.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.19.ffn_norm.weight                        -> blk.19.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.20.attention.wq.weight                    -> blk.20.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.20.attention.wk.weight                    -> blk.20.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.20.attention.wv.weight                    -> blk.20.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.20.attention.wo.weight                    -> blk.20.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.20.feed_forward.w1.weight                 -> blk.20.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.20.feed_forward.w2.weight                 -> blk.20.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.20.feed_forward.w3.weight                 -> blk.20.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.20.attention_norm.weight                  -> blk.20.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.20.ffn_norm.weight                        -> blk.20.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.21.attention.wq.weight                    -> blk.21.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.21.attention.wk.weight                    -> blk.21.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.21.attention.wv.weight                    -> blk.21.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.21.attention.wo.weight                    -> blk.21.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.21.feed_forward.w1.weight                 -> blk.21.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.21.feed_forward.w2.weight                 -> blk.21.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.21.feed_forward.w3.weight                 -> blk.21.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.21.attention_norm.weight                  -> blk.21.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.21.ffn_norm.weight                        -> blk.21.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.22.attention.wq.weight                    -> blk.22.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.22.attention.wk.weight                    -> blk.22.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.22.attention.wv.weight                    -> blk.22.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.22.attention.wo.weight                    -> blk.22.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.22.feed_forward.w1.weight                 -> blk.22.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.22.feed_forward.w2.weight                 -> blk.22.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.22.feed_forward.w3.weight                 -> blk.22.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.22.attention_norm.weight                  -> blk.22.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.22.ffn_norm.weight                        -> blk.22.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.23.attention.wq.weight                    -> blk.23.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.23.attention.wk.weight                    -> blk.23.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.23.attention.wv.weight                    -> blk.23.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.23.attention.wo.weight                    -> blk.23.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.23.feed_forward.w1.weight                 -> blk.23.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.23.feed_forward.w2.weight                 -> blk.23.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.23.feed_forward.w3.weight                 -> blk.23.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.23.attention_norm.weight                  -> blk.23.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.23.ffn_norm.weight                        -> blk.23.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.24.attention.wq.weight                    -> blk.24.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.24.attention.wk.weight                    -> blk.24.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.24.attention.wv.weight                    -> blk.24.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.24.attention.wo.weight                    -> blk.24.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.24.feed_forward.w1.weight                 -> blk.24.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.24.feed_forward.w2.weight                 -> blk.24.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.24.feed_forward.w3.weight                 -> blk.24.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.24.attention_norm.weight                  -> blk.24.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.24.ffn_norm.weight                        -> blk.24.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.25.attention.wq.weight                    -> blk.25.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.25.attention.wk.weight                    -> blk.25.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.25.attention.wv.weight                    -> blk.25.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.25.attention.wo.weight                    -> blk.25.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.25.feed_forward.w1.weight                 -> blk.25.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.25.feed_forward.w2.weight                 -> blk.25.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.25.feed_forward.w3.weight                 -> blk.25.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.25.attention_norm.weight                  -> blk.25.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.25.ffn_norm.weight                        -> blk.25.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.26.attention.wq.weight                    -> blk.26.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.26.attention.wk.weight                    -> blk.26.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.26.attention.wv.weight                    -> blk.26.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.26.attention.wo.weight                    -> blk.26.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.26.feed_forward.w1.weight                 -> blk.26.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.26.feed_forward.w2.weight                 -> blk.26.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.26.feed_forward.w3.weight                 -> blk.26.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.26.attention_norm.weight                  -> blk.26.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.26.ffn_norm.weight                        -> blk.26.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.27.attention.wq.weight                    -> blk.27.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.27.attention.wk.weight                    -> blk.27.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.27.attention.wv.weight                    -> blk.27.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.27.attention.wo.weight                    -> blk.27.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.27.feed_forward.w1.weight                 -> blk.27.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.27.feed_forward.w2.weight                 -> blk.27.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.27.feed_forward.w3.weight                 -> blk.27.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.27.attention_norm.weight                  -> blk.27.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.27.ffn_norm.weight                        -> blk.27.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.28.attention.wq.weight                    -> blk.28.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.28.attention.wk.weight                    -> blk.28.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.28.attention.wv.weight                    -> blk.28.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.28.attention.wo.weight                    -> blk.28.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.28.feed_forward.w1.weight                 -> blk.28.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.28.feed_forward.w2.weight                 -> blk.28.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.28.feed_forward.w3.weight                 -> blk.28.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.28.attention_norm.weight                  -> blk.28.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.28.ffn_norm.weight                        -> blk.28.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.29.attention.wq.weight                    -> blk.29.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.29.attention.wk.weight                    -> blk.29.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.29.attention.wv.weight                    -> blk.29.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.29.attention.wo.weight                    -> blk.29.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.29.feed_forward.w1.weight                 -> blk.29.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.29.feed_forward.w2.weight                 -> blk.29.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.29.feed_forward.w3.weight                 -> blk.29.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.29.attention_norm.weight                  -> blk.29.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.29.ffn_norm.weight                        -> blk.29.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.30.attention.wq.weight                    -> blk.30.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.30.attention.wk.weight                    -> blk.30.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.30.attention.wv.weight                    -> blk.30.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.30.attention.wo.weight                    -> blk.30.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.30.feed_forward.w1.weight                 -> blk.30.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.30.feed_forward.w2.weight                 -> blk.30.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.30.feed_forward.w3.weight                 -> blk.30.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.30.attention_norm.weight                  -> blk.30.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.30.ffn_norm.weight                        -> blk.30.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.31.attention.wq.weight                    -> blk.31.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.31.attention.wk.weight                    -> blk.31.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.31.attention.wv.weight                    -> blk.31.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.31.attention.wo.weight                    -> blk.31.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.31.feed_forward.w1.weight                 -> blk.31.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.31.feed_forward.w2.weight                 -> blk.31.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.31.feed_forward.w3.weight                 -> blk.31.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.31.attention_norm.weight                  -> blk.31.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.31.ffn_norm.weight                        -> blk.31.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.32.attention.wq.weight                    -> blk.32.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.32.attention.wk.weight                    -> blk.32.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.32.attention.wv.weight                    -> blk.32.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.32.attention.wo.weight                    -> blk.32.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.32.feed_forward.w1.weight                 -> blk.32.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.32.feed_forward.w2.weight                 -> blk.32.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.32.feed_forward.w3.weight                 -> blk.32.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.32.attention_norm.weight                  -> blk.32.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.32.ffn_norm.weight                        -> blk.32.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.33.attention.wq.weight                    -> blk.33.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.33.attention.wk.weight                    -> blk.33.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.33.attention.wv.weight                    -> blk.33.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.33.attention.wo.weight                    -> blk.33.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.33.feed_forward.w1.weight                 -> blk.33.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.33.feed_forward.w2.weight                 -> blk.33.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.33.feed_forward.w3.weight                 -> blk.33.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.33.attention_norm.weight                  -> blk.33.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.33.ffn_norm.weight                        -> blk.33.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.34.attention.wq.weight                    -> blk.34.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.34.attention.wk.weight                    -> blk.34.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.34.attention.wv.weight                    -> blk.34.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.34.attention.wo.weight                    -> blk.34.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.34.feed_forward.w1.weight                 -> blk.34.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.34.feed_forward.w2.weight                 -> blk.34.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.34.feed_forward.w3.weight                 -> blk.34.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.34.attention_norm.weight                  -> blk.34.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.34.ffn_norm.weight                        -> blk.34.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.35.attention.wq.weight                    -> blk.35.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.35.attention.wk.weight                    -> blk.35.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.35.attention.wv.weight                    -> blk.35.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.35.attention.wo.weight                    -> blk.35.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.35.feed_forward.w1.weight                 -> blk.35.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.35.feed_forward.w2.weight                 -> blk.35.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.35.feed_forward.w3.weight                 -> blk.35.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.35.attention_norm.weight                  -> blk.35.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.35.ffn_norm.weight                        -> blk.35.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.36.attention.wq.weight                    -> blk.36.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.36.attention.wk.weight                    -> blk.36.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.36.attention.wv.weight                    -> blk.36.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.36.attention.wo.weight                    -> blk.36.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.36.feed_forward.w1.weight                 -> blk.36.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.36.feed_forward.w2.weight                 -> blk.36.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.36.feed_forward.w3.weight                 -> blk.36.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.36.attention_norm.weight                  -> blk.36.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.36.ffn_norm.weight                        -> blk.36.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.37.attention.wq.weight                    -> blk.37.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.37.attention.wk.weight                    -> blk.37.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.37.attention.wv.weight                    -> blk.37.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.37.attention.wo.weight                    -> blk.37.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.37.feed_forward.w1.weight                 -> blk.37.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.37.feed_forward.w2.weight                 -> blk.37.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.37.feed_forward.w3.weight                 -> blk.37.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.37.attention_norm.weight                  -> blk.37.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.37.ffn_norm.weight                        -> blk.37.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.38.attention.wq.weight                    -> blk.38.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.38.attention.wk.weight                    -> blk.38.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.38.attention.wv.weight                    -> blk.38.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.38.attention.wo.weight                    -> blk.38.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.38.feed_forward.w1.weight                 -> blk.38.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.38.feed_forward.w2.weight                 -> blk.38.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.38.feed_forward.w3.weight                 -> blk.38.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.38.attention_norm.weight                  -> blk.38.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.38.ffn_norm.weight                        -> blk.38.ffn_norm.weight                   | BF16   | [5120]\n",
      "layers.39.attention.wq.weight                    -> blk.39.attn_q.weight                     | BF16   | [5120, 5120]\n",
      "layers.39.attention.wk.weight                    -> blk.39.attn_k.weight                     | BF16   | [5120, 5120]\n",
      "layers.39.attention.wv.weight                    -> blk.39.attn_v.weight                     | BF16   | [5120, 5120]\n",
      "layers.39.attention.wo.weight                    -> blk.39.attn_output.weight                | BF16   | [5120, 5120]\n",
      "layers.39.feed_forward.w1.weight                 -> blk.39.ffn_gate.weight                   | BF16   | [13824, 5120]\n",
      "layers.39.feed_forward.w2.weight                 -> blk.39.ffn_down.weight                   | BF16   | [5120, 13824]\n",
      "layers.39.feed_forward.w3.weight                 -> blk.39.ffn_up.weight                     | BF16   | [13824, 5120]\n",
      "layers.39.attention_norm.weight                  -> blk.39.attn_norm.weight                  | BF16   | [5120]\n",
      "layers.39.ffn_norm.weight                        -> blk.39.ffn_norm.weight                   | BF16   | [5120]\n",
      "skipping tensor rope_freqs\n",
      "Writing models/13B-v2/ggml-model-f16.gguf, format 1\n",
      "gguf: This GGUF file is for Little Endian only\n",
      "gguf: Adding 61249 merge(s).\n",
      "gguf: Setting special token type bos to 1\n",
      "gguf: Setting special token type eos to 2\n",
      "gguf: Setting special token type unk to 0\n",
      "gguf: Setting add_bos_token to True\n",
      "gguf: Setting add_eos_token to False\n",
      "[  1/363] Writing tensor token_embd.weight                      | size  32000 x   5120  | type F16  | T+   1\n",
      "[  2/363] Writing tensor output_norm.weight                     | size   5120           | type F32  | T+   1\n",
      "[  3/363] Writing tensor output.weight                          | size  32000 x   5120  | type F16  | T+   1\n",
      "[  4/363] Writing tensor blk.0.attn_q.weight                    | size   5120 x   5120  | type F16  | T+   1\n",
      "[  5/363] Writing tensor blk.0.attn_k.weight                    | size   5120 x   5120  | type F16  | T+   1\n",
      "[  6/363] Writing tensor blk.0.attn_v.weight                    | size   5120 x   5120  | type F16  | T+   1\n",
      "[  7/363] Writing tensor blk.0.attn_output.weight               | size   5120 x   5120  | type F16  | T+   1\n",
      "[  8/363] Writing tensor blk.0.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+   1\n",
      "[  9/363] Writing tensor blk.0.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+   2\n",
      "[ 10/363] Writing tensor blk.0.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+   2\n",
      "[ 11/363] Writing tensor blk.0.attn_norm.weight                 | size   5120           | type F32  | T+   2\n",
      "[ 12/363] Writing tensor blk.0.ffn_norm.weight                  | size   5120           | type F32  | T+   2\n",
      "[ 13/363] Writing tensor blk.1.attn_q.weight                    | size   5120 x   5120  | type F16  | T+   2\n",
      "[ 14/363] Writing tensor blk.1.attn_k.weight                    | size   5120 x   5120  | type F16  | T+   2\n",
      "[ 15/363] Writing tensor blk.1.attn_v.weight                    | size   5120 x   5120  | type F16  | T+   2\n",
      "[ 16/363] Writing tensor blk.1.attn_output.weight               | size   5120 x   5120  | type F16  | T+   2\n",
      "[ 17/363] Writing tensor blk.1.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+   2\n",
      "[ 18/363] Writing tensor blk.1.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+   3\n",
      "[ 19/363] Writing tensor blk.1.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+   3\n",
      "[ 20/363] Writing tensor blk.1.attn_norm.weight                 | size   5120           | type F32  | T+   3\n",
      "[ 21/363] Writing tensor blk.1.ffn_norm.weight                  | size   5120           | type F32  | T+   3\n",
      "[ 22/363] Writing tensor blk.2.attn_q.weight                    | size   5120 x   5120  | type F16  | T+   3\n",
      "[ 23/363] Writing tensor blk.2.attn_k.weight                    | size   5120 x   5120  | type F16  | T+   3\n",
      "[ 24/363] Writing tensor blk.2.attn_v.weight                    | size   5120 x   5120  | type F16  | T+   3\n",
      "[ 25/363] Writing tensor blk.2.attn_output.weight               | size   5120 x   5120  | type F16  | T+   3\n",
      "[ 26/363] Writing tensor blk.2.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+   4\n",
      "[ 27/363] Writing tensor blk.2.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+   4\n",
      "[ 28/363] Writing tensor blk.2.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+   5\n",
      "[ 29/363] Writing tensor blk.2.attn_norm.weight                 | size   5120           | type F32  | T+   5\n",
      "[ 30/363] Writing tensor blk.2.ffn_norm.weight                  | size   5120           | type F32  | T+   5\n",
      "[ 31/363] Writing tensor blk.3.attn_q.weight                    | size   5120 x   5120  | type F16  | T+   5\n",
      "[ 32/363] Writing tensor blk.3.attn_k.weight                    | size   5120 x   5120  | type F16  | T+   5\n",
      "[ 33/363] Writing tensor blk.3.attn_v.weight                    | size   5120 x   5120  | type F16  | T+   5\n",
      "[ 34/363] Writing tensor blk.3.attn_output.weight               | size   5120 x   5120  | type F16  | T+   5\n",
      "[ 35/363] Writing tensor blk.3.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+   5\n",
      "[ 36/363] Writing tensor blk.3.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+   6\n",
      "[ 37/363] Writing tensor blk.3.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+   6\n",
      "[ 38/363] Writing tensor blk.3.attn_norm.weight                 | size   5120           | type F32  | T+   6\n",
      "[ 39/363] Writing tensor blk.3.ffn_norm.weight                  | size   5120           | type F32  | T+   6\n",
      "[ 40/363] Writing tensor blk.4.attn_q.weight                    | size   5120 x   5120  | type F16  | T+   6\n",
      "[ 41/363] Writing tensor blk.4.attn_k.weight                    | size   5120 x   5120  | type F16  | T+   6\n",
      "[ 42/363] Writing tensor blk.4.attn_v.weight                    | size   5120 x   5120  | type F16  | T+   6\n",
      "[ 43/363] Writing tensor blk.4.attn_output.weight               | size   5120 x   5120  | type F16  | T+   6\n",
      "[ 44/363] Writing tensor blk.4.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+   6\n",
      "[ 45/363] Writing tensor blk.4.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+   7\n",
      "[ 46/363] Writing tensor blk.4.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+   7\n",
      "[ 47/363] Writing tensor blk.4.attn_norm.weight                 | size   5120           | type F32  | T+   7\n",
      "[ 48/363] Writing tensor blk.4.ffn_norm.weight                  | size   5120           | type F32  | T+   7\n",
      "[ 49/363] Writing tensor blk.5.attn_q.weight                    | size   5120 x   5120  | type F16  | T+   7\n",
      "[ 50/363] Writing tensor blk.5.attn_k.weight                    | size   5120 x   5120  | type F16  | T+   7\n",
      "[ 51/363] Writing tensor blk.5.attn_v.weight                    | size   5120 x   5120  | type F16  | T+   7\n",
      "[ 52/363] Writing tensor blk.5.attn_output.weight               | size   5120 x   5120  | type F16  | T+   7\n",
      "[ 53/363] Writing tensor blk.5.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+   8\n",
      "[ 54/363] Writing tensor blk.5.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+   8\n",
      "[ 55/363] Writing tensor blk.5.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+   8\n",
      "[ 56/363] Writing tensor blk.5.attn_norm.weight                 | size   5120           | type F32  | T+   8\n",
      "[ 57/363] Writing tensor blk.5.ffn_norm.weight                  | size   5120           | type F32  | T+   8\n",
      "[ 58/363] Writing tensor blk.6.attn_q.weight                    | size   5120 x   5120  | type F16  | T+   8\n",
      "[ 59/363] Writing tensor blk.6.attn_k.weight                    | size   5120 x   5120  | type F16  | T+   8\n",
      "[ 60/363] Writing tensor blk.6.attn_v.weight                    | size   5120 x   5120  | type F16  | T+   8\n",
      "[ 61/363] Writing tensor blk.6.attn_output.weight               | size   5120 x   5120  | type F16  | T+   8\n",
      "[ 62/363] Writing tensor blk.6.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+   9\n",
      "[ 63/363] Writing tensor blk.6.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+   9\n",
      "[ 64/363] Writing tensor blk.6.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+   9\n",
      "[ 65/363] Writing tensor blk.6.attn_norm.weight                 | size   5120           | type F32  | T+   9\n",
      "[ 66/363] Writing tensor blk.6.ffn_norm.weight                  | size   5120           | type F32  | T+   9\n",
      "[ 67/363] Writing tensor blk.7.attn_q.weight                    | size   5120 x   5120  | type F16  | T+   9\n",
      "[ 68/363] Writing tensor blk.7.attn_k.weight                    | size   5120 x   5120  | type F16  | T+  10\n",
      "[ 69/363] Writing tensor blk.7.attn_v.weight                    | size   5120 x   5120  | type F16  | T+  10\n",
      "[ 70/363] Writing tensor blk.7.attn_output.weight               | size   5120 x   5120  | type F16  | T+  10\n",
      "[ 71/363] Writing tensor blk.7.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+  10\n",
      "[ 72/363] Writing tensor blk.7.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+  10\n",
      "[ 73/363] Writing tensor blk.7.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+  11\n",
      "[ 74/363] Writing tensor blk.7.attn_norm.weight                 | size   5120           | type F32  | T+  11\n",
      "[ 75/363] Writing tensor blk.7.ffn_norm.weight                  | size   5120           | type F32  | T+  11\n",
      "[ 76/363] Writing tensor blk.8.attn_q.weight                    | size   5120 x   5120  | type F16  | T+  11\n",
      "[ 77/363] Writing tensor blk.8.attn_k.weight                    | size   5120 x   5120  | type F16  | T+  11\n",
      "[ 78/363] Writing tensor blk.8.attn_v.weight                    | size   5120 x   5120  | type F16  | T+  11\n",
      "[ 79/363] Writing tensor blk.8.attn_output.weight               | size   5120 x   5120  | type F16  | T+  11\n",
      "[ 80/363] Writing tensor blk.8.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+  11\n",
      "[ 81/363] Writing tensor blk.8.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+  12\n",
      "[ 82/363] Writing tensor blk.8.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+  12\n",
      "[ 83/363] Writing tensor blk.8.attn_norm.weight                 | size   5120           | type F32  | T+  12\n",
      "[ 84/363] Writing tensor blk.8.ffn_norm.weight                  | size   5120           | type F32  | T+  12\n",
      "[ 85/363] Writing tensor blk.9.attn_q.weight                    | size   5120 x   5120  | type F16  | T+  12\n",
      "[ 86/363] Writing tensor blk.9.attn_k.weight                    | size   5120 x   5120  | type F16  | T+  12\n",
      "[ 87/363] Writing tensor blk.9.attn_v.weight                    | size   5120 x   5120  | type F16  | T+  12\n",
      "[ 88/363] Writing tensor blk.9.attn_output.weight               | size   5120 x   5120  | type F16  | T+  12\n",
      "[ 89/363] Writing tensor blk.9.ffn_gate.weight                  | size  13824 x   5120  | type F16  | T+  12\n",
      "[ 90/363] Writing tensor blk.9.ffn_down.weight                  | size   5120 x  13824  | type F16  | T+  13\n",
      "[ 91/363] Writing tensor blk.9.ffn_up.weight                    | size  13824 x   5120  | type F16  | T+  13\n",
      "[ 92/363] Writing tensor blk.9.attn_norm.weight                 | size   5120           | type F32  | T+  13\n",
      "[ 93/363] Writing tensor blk.9.ffn_norm.weight                  | size   5120           | type F32  | T+  13\n",
      "[ 94/363] Writing tensor blk.10.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  13\n",
      "[ 95/363] Writing tensor blk.10.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  13\n",
      "[ 96/363] Writing tensor blk.10.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  13\n",
      "[ 97/363] Writing tensor blk.10.attn_output.weight              | size   5120 x   5120  | type F16  | T+  13\n",
      "[ 98/363] Writing tensor blk.10.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  14\n",
      "[ 99/363] Writing tensor blk.10.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  14\n",
      "[100/363] Writing tensor blk.10.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  14\n",
      "[101/363] Writing tensor blk.10.attn_norm.weight                | size   5120           | type F32  | T+  14\n",
      "[102/363] Writing tensor blk.10.ffn_norm.weight                 | size   5120           | type F32  | T+  14\n",
      "[103/363] Writing tensor blk.11.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  14\n",
      "[104/363] Writing tensor blk.11.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  14\n",
      "[105/363] Writing tensor blk.11.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  14\n",
      "[106/363] Writing tensor blk.11.attn_output.weight              | size   5120 x   5120  | type F16  | T+  14\n",
      "[107/363] Writing tensor blk.11.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  15\n",
      "[108/363] Writing tensor blk.11.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  15\n",
      "[109/363] Writing tensor blk.11.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  16\n",
      "[110/363] Writing tensor blk.11.attn_norm.weight                | size   5120           | type F32  | T+  16\n",
      "[111/363] Writing tensor blk.11.ffn_norm.weight                 | size   5120           | type F32  | T+  16\n",
      "[112/363] Writing tensor blk.12.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  16\n",
      "[113/363] Writing tensor blk.12.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  16\n",
      "[114/363] Writing tensor blk.12.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  16\n",
      "[115/363] Writing tensor blk.12.attn_output.weight              | size   5120 x   5120  | type F16  | T+  16\n",
      "[116/363] Writing tensor blk.12.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  16\n",
      "[117/363] Writing tensor blk.12.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  17\n",
      "[118/363] Writing tensor blk.12.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  17\n",
      "[119/363] Writing tensor blk.12.attn_norm.weight                | size   5120           | type F32  | T+  17\n",
      "[120/363] Writing tensor blk.12.ffn_norm.weight                 | size   5120           | type F32  | T+  17\n",
      "[121/363] Writing tensor blk.13.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  17\n",
      "[122/363] Writing tensor blk.13.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  17\n",
      "[123/363] Writing tensor blk.13.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  17\n",
      "[124/363] Writing tensor blk.13.attn_output.weight              | size   5120 x   5120  | type F16  | T+  17\n",
      "[125/363] Writing tensor blk.13.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  17\n",
      "[126/363] Writing tensor blk.13.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  18\n",
      "[127/363] Writing tensor blk.13.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  18\n",
      "[128/363] Writing tensor blk.13.attn_norm.weight                | size   5120           | type F32  | T+  18\n",
      "[129/363] Writing tensor blk.13.ffn_norm.weight                 | size   5120           | type F32  | T+  18\n",
      "[130/363] Writing tensor blk.14.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  18\n",
      "[131/363] Writing tensor blk.14.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  18\n",
      "[132/363] Writing tensor blk.14.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  18\n",
      "[133/363] Writing tensor blk.14.attn_output.weight              | size   5120 x   5120  | type F16  | T+  18\n",
      "[134/363] Writing tensor blk.14.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  19\n",
      "[135/363] Writing tensor blk.14.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  19\n",
      "[136/363] Writing tensor blk.14.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  19\n",
      "[137/363] Writing tensor blk.14.attn_norm.weight                | size   5120           | type F32  | T+  19\n",
      "[138/363] Writing tensor blk.14.ffn_norm.weight                 | size   5120           | type F32  | T+  19\n",
      "[139/363] Writing tensor blk.15.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  19\n",
      "[140/363] Writing tensor blk.15.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  19\n",
      "[141/363] Writing tensor blk.15.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  19\n",
      "[142/363] Writing tensor blk.15.attn_output.weight              | size   5120 x   5120  | type F16  | T+  19\n",
      "[143/363] Writing tensor blk.15.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  20\n",
      "[144/363] Writing tensor blk.15.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  20\n",
      "[145/363] Writing tensor blk.15.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  21\n",
      "[146/363] Writing tensor blk.15.attn_norm.weight                | size   5120           | type F32  | T+  21\n",
      "[147/363] Writing tensor blk.15.ffn_norm.weight                 | size   5120           | type F32  | T+  21\n",
      "[148/363] Writing tensor blk.16.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  21\n",
      "[149/363] Writing tensor blk.16.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  21\n",
      "[150/363] Writing tensor blk.16.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  21\n",
      "[151/363] Writing tensor blk.16.attn_output.weight              | size   5120 x   5120  | type F16  | T+  21\n",
      "[152/363] Writing tensor blk.16.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  21\n",
      "[153/363] Writing tensor blk.16.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  22\n",
      "[154/363] Writing tensor blk.16.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  22\n",
      "[155/363] Writing tensor blk.16.attn_norm.weight                | size   5120           | type F32  | T+  22\n",
      "[156/363] Writing tensor blk.16.ffn_norm.weight                 | size   5120           | type F32  | T+  22\n",
      "[157/363] Writing tensor blk.17.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  22\n",
      "[158/363] Writing tensor blk.17.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  22\n",
      "[159/363] Writing tensor blk.17.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  22\n",
      "[160/363] Writing tensor blk.17.attn_output.weight              | size   5120 x   5120  | type F16  | T+  22\n",
      "[161/363] Writing tensor blk.17.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  22\n",
      "[162/363] Writing tensor blk.17.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  23\n",
      "[163/363] Writing tensor blk.17.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  23\n",
      "[164/363] Writing tensor blk.17.attn_norm.weight                | size   5120           | type F32  | T+  23\n",
      "[165/363] Writing tensor blk.17.ffn_norm.weight                 | size   5120           | type F32  | T+  23\n",
      "[166/363] Writing tensor blk.18.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  23\n",
      "[167/363] Writing tensor blk.18.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  23\n",
      "[168/363] Writing tensor blk.18.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  23\n",
      "[169/363] Writing tensor blk.18.attn_output.weight              | size   5120 x   5120  | type F16  | T+  23\n",
      "[170/363] Writing tensor blk.18.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  24\n",
      "[171/363] Writing tensor blk.18.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  24\n",
      "[172/363] Writing tensor blk.18.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  24\n",
      "[173/363] Writing tensor blk.18.attn_norm.weight                | size   5120           | type F32  | T+  24\n",
      "[174/363] Writing tensor blk.18.ffn_norm.weight                 | size   5120           | type F32  | T+  24\n",
      "[175/363] Writing tensor blk.19.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  24\n",
      "[176/363] Writing tensor blk.19.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  24\n",
      "[177/363] Writing tensor blk.19.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  24\n",
      "[178/363] Writing tensor blk.19.attn_output.weight              | size   5120 x   5120  | type F16  | T+  24\n",
      "[179/363] Writing tensor blk.19.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  25\n",
      "[180/363] Writing tensor blk.19.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  25\n",
      "[181/363] Writing tensor blk.19.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  26\n",
      "[182/363] Writing tensor blk.19.attn_norm.weight                | size   5120           | type F32  | T+  26\n",
      "[183/363] Writing tensor blk.19.ffn_norm.weight                 | size   5120           | type F32  | T+  26\n",
      "[184/363] Writing tensor blk.20.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  26\n",
      "[185/363] Writing tensor blk.20.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  26\n",
      "[186/363] Writing tensor blk.20.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  26\n",
      "[187/363] Writing tensor blk.20.attn_output.weight              | size   5120 x   5120  | type F16  | T+  26\n",
      "[188/363] Writing tensor blk.20.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  26\n",
      "[189/363] Writing tensor blk.20.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  27\n",
      "[190/363] Writing tensor blk.20.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  27\n",
      "[191/363] Writing tensor blk.20.attn_norm.weight                | size   5120           | type F32  | T+  27\n",
      "[192/363] Writing tensor blk.20.ffn_norm.weight                 | size   5120           | type F32  | T+  27\n",
      "[193/363] Writing tensor blk.21.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  27\n",
      "[194/363] Writing tensor blk.21.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  27\n",
      "[195/363] Writing tensor blk.21.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  27\n",
      "[196/363] Writing tensor blk.21.attn_output.weight              | size   5120 x   5120  | type F16  | T+  27\n",
      "[197/363] Writing tensor blk.21.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  27\n",
      "[198/363] Writing tensor blk.21.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  28\n",
      "[199/363] Writing tensor blk.21.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  28\n",
      "[200/363] Writing tensor blk.21.attn_norm.weight                | size   5120           | type F32  | T+  28\n",
      "[201/363] Writing tensor blk.21.ffn_norm.weight                 | size   5120           | type F32  | T+  28\n",
      "[202/363] Writing tensor blk.22.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  28\n",
      "[203/363] Writing tensor blk.22.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  28\n",
      "[204/363] Writing tensor blk.22.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  28\n",
      "[205/363] Writing tensor blk.22.attn_output.weight              | size   5120 x   5120  | type F16  | T+  28\n",
      "[206/363] Writing tensor blk.22.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  29\n",
      "[207/363] Writing tensor blk.22.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  29\n",
      "[208/363] Writing tensor blk.22.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  29\n",
      "[209/363] Writing tensor blk.22.attn_norm.weight                | size   5120           | type F32  | T+  29\n",
      "[210/363] Writing tensor blk.22.ffn_norm.weight                 | size   5120           | type F32  | T+  29\n",
      "[211/363] Writing tensor blk.23.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  29\n",
      "[212/363] Writing tensor blk.23.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  29\n",
      "[213/363] Writing tensor blk.23.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  29\n",
      "[214/363] Writing tensor blk.23.attn_output.weight              | size   5120 x   5120  | type F16  | T+  29\n",
      "[215/363] Writing tensor blk.23.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  30\n",
      "[216/363] Writing tensor blk.23.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  30\n",
      "[217/363] Writing tensor blk.23.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  30\n",
      "[218/363] Writing tensor blk.23.attn_norm.weight                | size   5120           | type F32  | T+  30\n",
      "[219/363] Writing tensor blk.23.ffn_norm.weight                 | size   5120           | type F32  | T+  30\n",
      "[220/363] Writing tensor blk.24.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  30\n",
      "[221/363] Writing tensor blk.24.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  31\n",
      "[222/363] Writing tensor blk.24.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  31\n",
      "[223/363] Writing tensor blk.24.attn_output.weight              | size   5120 x   5120  | type F16  | T+  31\n",
      "[224/363] Writing tensor blk.24.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  31\n",
      "[225/363] Writing tensor blk.24.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  31\n",
      "[226/363] Writing tensor blk.24.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  32\n",
      "[227/363] Writing tensor blk.24.attn_norm.weight                | size   5120           | type F32  | T+  32\n",
      "[228/363] Writing tensor blk.24.ffn_norm.weight                 | size   5120           | type F32  | T+  32\n",
      "[229/363] Writing tensor blk.25.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  32\n",
      "[230/363] Writing tensor blk.25.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  32\n",
      "[231/363] Writing tensor blk.25.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  32\n",
      "[232/363] Writing tensor blk.25.attn_output.weight              | size   5120 x   5120  | type F16  | T+  32\n",
      "[233/363] Writing tensor blk.25.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  32\n",
      "[234/363] Writing tensor blk.25.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  33\n",
      "[235/363] Writing tensor blk.25.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  33\n",
      "[236/363] Writing tensor blk.25.attn_norm.weight                | size   5120           | type F32  | T+  33\n",
      "[237/363] Writing tensor blk.25.ffn_norm.weight                 | size   5120           | type F32  | T+  33\n",
      "[238/363] Writing tensor blk.26.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  33\n",
      "[239/363] Writing tensor blk.26.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  33\n",
      "[240/363] Writing tensor blk.26.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  33\n",
      "[241/363] Writing tensor blk.26.attn_output.weight              | size   5120 x   5120  | type F16  | T+  33\n",
      "[242/363] Writing tensor blk.26.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  33\n",
      "[243/363] Writing tensor blk.26.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  34\n",
      "[244/363] Writing tensor blk.26.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  34\n",
      "[245/363] Writing tensor blk.26.attn_norm.weight                | size   5120           | type F32  | T+  34\n",
      "[246/363] Writing tensor blk.26.ffn_norm.weight                 | size   5120           | type F32  | T+  34\n",
      "[247/363] Writing tensor blk.27.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  34\n",
      "[248/363] Writing tensor blk.27.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  34\n",
      "[249/363] Writing tensor blk.27.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  34\n",
      "[250/363] Writing tensor blk.27.attn_output.weight              | size   5120 x   5120  | type F16  | T+  34\n",
      "[251/363] Writing tensor blk.27.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  35\n",
      "[252/363] Writing tensor blk.27.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  35\n",
      "[253/363] Writing tensor blk.27.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  35\n",
      "[254/363] Writing tensor blk.27.attn_norm.weight                | size   5120           | type F32  | T+  35\n",
      "[255/363] Writing tensor blk.27.ffn_norm.weight                 | size   5120           | type F32  | T+  35\n",
      "[256/363] Writing tensor blk.28.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  35\n",
      "[257/363] Writing tensor blk.28.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  35\n",
      "[258/363] Writing tensor blk.28.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  35\n",
      "[259/363] Writing tensor blk.28.attn_output.weight              | size   5120 x   5120  | type F16  | T+  35\n",
      "[260/363] Writing tensor blk.28.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  36\n",
      "[261/363] Writing tensor blk.28.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  36\n",
      "[262/363] Writing tensor blk.28.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  36\n",
      "[263/363] Writing tensor blk.28.attn_norm.weight                | size   5120           | type F32  | T+  37\n",
      "[264/363] Writing tensor blk.28.ffn_norm.weight                 | size   5120           | type F32  | T+  37\n",
      "[265/363] Writing tensor blk.29.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  37\n",
      "[266/363] Writing tensor blk.29.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  37\n",
      "[267/363] Writing tensor blk.29.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  37\n",
      "[268/363] Writing tensor blk.29.attn_output.weight              | size   5120 x   5120  | type F16  | T+  37\n",
      "[269/363] Writing tensor blk.29.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  37\n",
      "[270/363] Writing tensor blk.29.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  37\n",
      "[271/363] Writing tensor blk.29.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  38\n",
      "[272/363] Writing tensor blk.29.attn_norm.weight                | size   5120           | type F32  | T+  38\n",
      "[273/363] Writing tensor blk.29.ffn_norm.weight                 | size   5120           | type F32  | T+  38\n",
      "[274/363] Writing tensor blk.30.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  38\n",
      "[275/363] Writing tensor blk.30.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  38\n",
      "[276/363] Writing tensor blk.30.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  38\n",
      "[277/363] Writing tensor blk.30.attn_output.weight              | size   5120 x   5120  | type F16  | T+  38\n",
      "[278/363] Writing tensor blk.30.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  38\n",
      "[279/363] Writing tensor blk.30.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  39\n",
      "[280/363] Writing tensor blk.30.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  39\n",
      "[281/363] Writing tensor blk.30.attn_norm.weight                | size   5120           | type F32  | T+  39\n",
      "[282/363] Writing tensor blk.30.ffn_norm.weight                 | size   5120           | type F32  | T+  39\n",
      "[283/363] Writing tensor blk.31.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  39\n",
      "[284/363] Writing tensor blk.31.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  39\n",
      "[285/363] Writing tensor blk.31.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  39\n",
      "[286/363] Writing tensor blk.31.attn_output.weight              | size   5120 x   5120  | type F16  | T+  39\n",
      "[287/363] Writing tensor blk.31.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  39\n",
      "[288/363] Writing tensor blk.31.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  40\n",
      "[289/363] Writing tensor blk.31.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  40\n",
      "[290/363] Writing tensor blk.31.attn_norm.weight                | size   5120           | type F32  | T+  40\n",
      "[291/363] Writing tensor blk.31.ffn_norm.weight                 | size   5120           | type F32  | T+  40\n",
      "[292/363] Writing tensor blk.32.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  40\n",
      "[293/363] Writing tensor blk.32.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  40\n",
      "[294/363] Writing tensor blk.32.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  40\n",
      "[295/363] Writing tensor blk.32.attn_output.weight              | size   5120 x   5120  | type F16  | T+  40\n",
      "[296/363] Writing tensor blk.32.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  41\n",
      "[297/363] Writing tensor blk.32.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  41\n",
      "[298/363] Writing tensor blk.32.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  41\n",
      "[299/363] Writing tensor blk.32.attn_norm.weight                | size   5120           | type F32  | T+  41\n",
      "[300/363] Writing tensor blk.32.ffn_norm.weight                 | size   5120           | type F32  | T+  41\n",
      "[301/363] Writing tensor blk.33.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  41\n",
      "[302/363] Writing tensor blk.33.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  41\n",
      "[303/363] Writing tensor blk.33.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  41\n",
      "[304/363] Writing tensor blk.33.attn_output.weight              | size   5120 x   5120  | type F16  | T+  41\n",
      "[305/363] Writing tensor blk.33.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  42\n",
      "[306/363] Writing tensor blk.33.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  42\n",
      "[307/363] Writing tensor blk.33.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  43\n",
      "[308/363] Writing tensor blk.33.attn_norm.weight                | size   5120           | type F32  | T+  43\n",
      "[309/363] Writing tensor blk.33.ffn_norm.weight                 | size   5120           | type F32  | T+  43\n",
      "[310/363] Writing tensor blk.34.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  43\n",
      "[311/363] Writing tensor blk.34.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  43\n",
      "[312/363] Writing tensor blk.34.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  43\n",
      "[313/363] Writing tensor blk.34.attn_output.weight              | size   5120 x   5120  | type F16  | T+  43\n",
      "[314/363] Writing tensor blk.34.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  43\n",
      "[315/363] Writing tensor blk.34.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  43\n",
      "[316/363] Writing tensor blk.34.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  44\n",
      "[317/363] Writing tensor blk.34.attn_norm.weight                | size   5120           | type F32  | T+  44\n",
      "[318/363] Writing tensor blk.34.ffn_norm.weight                 | size   5120           | type F32  | T+  44\n",
      "[319/363] Writing tensor blk.35.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  44\n",
      "[320/363] Writing tensor blk.35.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  44\n",
      "[321/363] Writing tensor blk.35.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  44\n",
      "[322/363] Writing tensor blk.35.attn_output.weight              | size   5120 x   5120  | type F16  | T+  44\n",
      "[323/363] Writing tensor blk.35.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  44\n",
      "[324/363] Writing tensor blk.35.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  45\n",
      "[325/363] Writing tensor blk.35.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  45\n",
      "[326/363] Writing tensor blk.35.attn_norm.weight                | size   5120           | type F32  | T+  45\n",
      "[327/363] Writing tensor blk.35.ffn_norm.weight                 | size   5120           | type F32  | T+  45\n",
      "[328/363] Writing tensor blk.36.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  45\n",
      "[329/363] Writing tensor blk.36.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  45\n",
      "[330/363] Writing tensor blk.36.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  45\n",
      "[331/363] Writing tensor blk.36.attn_output.weight              | size   5120 x   5120  | type F16  | T+  45\n",
      "[332/363] Writing tensor blk.36.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  45\n",
      "[333/363] Writing tensor blk.36.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  46\n",
      "[334/363] Writing tensor blk.36.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  46\n",
      "[335/363] Writing tensor blk.36.attn_norm.weight                | size   5120           | type F32  | T+  46\n",
      "[336/363] Writing tensor blk.36.ffn_norm.weight                 | size   5120           | type F32  | T+  46\n",
      "[337/363] Writing tensor blk.37.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  46\n",
      "[338/363] Writing tensor blk.37.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  46\n",
      "[339/363] Writing tensor blk.37.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  46\n",
      "[340/363] Writing tensor blk.37.attn_output.weight              | size   5120 x   5120  | type F16  | T+  46\n",
      "[341/363] Writing tensor blk.37.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  47\n",
      "[342/363] Writing tensor blk.37.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  47\n",
      "[343/363] Writing tensor blk.37.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  47\n",
      "[344/363] Writing tensor blk.37.attn_norm.weight                | size   5120           | type F32  | T+  47\n",
      "[345/363] Writing tensor blk.37.ffn_norm.weight                 | size   5120           | type F32  | T+  47\n",
      "[346/363] Writing tensor blk.38.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  47\n",
      "[347/363] Writing tensor blk.38.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  47\n",
      "[348/363] Writing tensor blk.38.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  48\n",
      "[349/363] Writing tensor blk.38.attn_output.weight              | size   5120 x   5120  | type F16  | T+  48\n",
      "[350/363] Writing tensor blk.38.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  48\n",
      "[351/363] Writing tensor blk.38.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  48\n",
      "[352/363] Writing tensor blk.38.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  49\n",
      "[353/363] Writing tensor blk.38.attn_norm.weight                | size   5120           | type F32  | T+  49\n",
      "[354/363] Writing tensor blk.38.ffn_norm.weight                 | size   5120           | type F32  | T+  49\n",
      "[355/363] Writing tensor blk.39.attn_q.weight                   | size   5120 x   5120  | type F16  | T+  49\n",
      "[356/363] Writing tensor blk.39.attn_k.weight                   | size   5120 x   5120  | type F16  | T+  49\n",
      "[357/363] Writing tensor blk.39.attn_v.weight                   | size   5120 x   5120  | type F16  | T+  49\n",
      "[358/363] Writing tensor blk.39.attn_output.weight              | size   5120 x   5120  | type F16  | T+  49\n",
      "[359/363] Writing tensor blk.39.ffn_gate.weight                 | size  13824 x   5120  | type F16  | T+  49\n",
      "[360/363] Writing tensor blk.39.ffn_down.weight                 | size   5120 x  13824  | type F16  | T+  50\n",
      "[361/363] Writing tensor blk.39.ffn_up.weight                   | size  13824 x   5120  | type F16  | T+  50\n",
      "[362/363] Writing tensor blk.39.attn_norm.weight                | size   5120           | type F32  | T+  50\n",
      "[363/363] Writing tensor blk.39.ffn_norm.weight                 | size   5120           | type F32  | T+  50\n",
      "Wrote models/13B-v2/ggml-model-f16.gguf\n",
      "Loading model file models/70B-v2/consolidated.00.pth\n",
      "Loading model file models/70B-v2/consolidated.01.pth\n",
      "Loading model file models/70B-v2/consolidated.02.pth\n",
      "Loading model file models/70B-v2/consolidated.03.pth\n",
      "Loading model file models/70B-v2/consolidated.04.pth\n",
      "Loading model file models/70B-v2/consolidated.05.pth\n",
      "Loading model file models/70B-v2/consolidated.06.pth\n",
      "Loading model file models/70B-v2/consolidated.07.pth\n",
      "params = Params(n_vocab=32000, n_embd=8192, n_layer=80, n_ctx=4096, n_ff=28672, n_head=64, n_head_kv=8, n_experts=None, n_experts_used=None, f_norm_eps=1e-05, rope_scaling_type=None, f_rope_freq_base=None, f_rope_scale=None, n_orig_ctx=None, rope_finetuned=None, ftype=None, path_model=PosixPath('models/70B-v2'))\n",
      "32000 32000\n",
      "Vocab info: <VocabLoader with 32000 base tokens and 0 added tokens>\n",
      "Special vocab info: <SpecialVocab with 61249 merges, special tokens {'bos': 1, 'eos': 2, 'unk': 0}, add special tokens {'bos': True, 'eos': False}>\n",
      "tok_embeddings.weight                            -> token_embd.weight                        | BF16   | [32000, 8192]\n",
      "norm.weight                                      -> output_norm.weight                       | BF16   | [8192]\n",
      "output.weight                                    -> output.weight                            | BF16   | [32000, 8192]\n",
      "layers.0.attention.wq.weight                     -> blk.0.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.0.attention.wk.weight                     -> blk.0.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.0.attention.wv.weight                     -> blk.0.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.0.attention.wo.weight                     -> blk.0.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.0.feed_forward.w1.weight                  -> blk.0.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.0.feed_forward.w2.weight                  -> blk.0.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.0.feed_forward.w3.weight                  -> blk.0.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.0.attention_norm.weight                   -> blk.0.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.0.ffn_norm.weight                         -> blk.0.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.1.attention.wq.weight                     -> blk.1.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.1.attention.wk.weight                     -> blk.1.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.1.attention.wv.weight                     -> blk.1.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.1.attention.wo.weight                     -> blk.1.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.1.feed_forward.w1.weight                  -> blk.1.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.1.feed_forward.w2.weight                  -> blk.1.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.1.feed_forward.w3.weight                  -> blk.1.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.1.attention_norm.weight                   -> blk.1.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.1.ffn_norm.weight                         -> blk.1.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.2.attention.wq.weight                     -> blk.2.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.2.attention.wk.weight                     -> blk.2.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.2.attention.wv.weight                     -> blk.2.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.2.attention.wo.weight                     -> blk.2.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.2.feed_forward.w1.weight                  -> blk.2.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.2.feed_forward.w2.weight                  -> blk.2.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.2.feed_forward.w3.weight                  -> blk.2.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.2.attention_norm.weight                   -> blk.2.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.2.ffn_norm.weight                         -> blk.2.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.3.attention.wq.weight                     -> blk.3.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.3.attention.wk.weight                     -> blk.3.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.3.attention.wv.weight                     -> blk.3.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.3.attention.wo.weight                     -> blk.3.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.3.feed_forward.w1.weight                  -> blk.3.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.3.feed_forward.w2.weight                  -> blk.3.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.3.feed_forward.w3.weight                  -> blk.3.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.3.attention_norm.weight                   -> blk.3.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.3.ffn_norm.weight                         -> blk.3.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.4.attention.wq.weight                     -> blk.4.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.4.attention.wk.weight                     -> blk.4.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.4.attention.wv.weight                     -> blk.4.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.4.attention.wo.weight                     -> blk.4.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.4.feed_forward.w1.weight                  -> blk.4.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.4.feed_forward.w2.weight                  -> blk.4.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.4.feed_forward.w3.weight                  -> blk.4.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.4.attention_norm.weight                   -> blk.4.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.4.ffn_norm.weight                         -> blk.4.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.5.attention.wq.weight                     -> blk.5.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.5.attention.wk.weight                     -> blk.5.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.5.attention.wv.weight                     -> blk.5.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.5.attention.wo.weight                     -> blk.5.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.5.feed_forward.w1.weight                  -> blk.5.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.5.feed_forward.w2.weight                  -> blk.5.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.5.feed_forward.w3.weight                  -> blk.5.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.5.attention_norm.weight                   -> blk.5.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.5.ffn_norm.weight                         -> blk.5.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.6.attention.wq.weight                     -> blk.6.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.6.attention.wk.weight                     -> blk.6.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.6.attention.wv.weight                     -> blk.6.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.6.attention.wo.weight                     -> blk.6.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.6.feed_forward.w1.weight                  -> blk.6.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.6.feed_forward.w2.weight                  -> blk.6.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.6.feed_forward.w3.weight                  -> blk.6.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.6.attention_norm.weight                   -> blk.6.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.6.ffn_norm.weight                         -> blk.6.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.7.attention.wq.weight                     -> blk.7.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.7.attention.wk.weight                     -> blk.7.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.7.attention.wv.weight                     -> blk.7.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.7.attention.wo.weight                     -> blk.7.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.7.feed_forward.w1.weight                  -> blk.7.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.7.feed_forward.w2.weight                  -> blk.7.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.7.feed_forward.w3.weight                  -> blk.7.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.7.attention_norm.weight                   -> blk.7.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.7.ffn_norm.weight                         -> blk.7.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.8.attention.wq.weight                     -> blk.8.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.8.attention.wk.weight                     -> blk.8.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.8.attention.wv.weight                     -> blk.8.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.8.attention.wo.weight                     -> blk.8.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.8.feed_forward.w1.weight                  -> blk.8.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.8.feed_forward.w2.weight                  -> blk.8.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.8.feed_forward.w3.weight                  -> blk.8.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.8.attention_norm.weight                   -> blk.8.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.8.ffn_norm.weight                         -> blk.8.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.9.attention.wq.weight                     -> blk.9.attn_q.weight                      | BF16   | [8192, 8192]\n",
      "layers.9.attention.wk.weight                     -> blk.9.attn_k.weight                      | BF16   | [1024, 8192]\n",
      "layers.9.attention.wv.weight                     -> blk.9.attn_v.weight                      | BF16   | [1024, 8192]\n",
      "layers.9.attention.wo.weight                     -> blk.9.attn_output.weight                 | BF16   | [8192, 8192]\n",
      "layers.9.feed_forward.w1.weight                  -> blk.9.ffn_gate.weight                    | BF16   | [28672, 8192]\n",
      "layers.9.feed_forward.w2.weight                  -> blk.9.ffn_down.weight                    | BF16   | [8192, 28672]\n",
      "layers.9.feed_forward.w3.weight                  -> blk.9.ffn_up.weight                      | BF16   | [28672, 8192]\n",
      "layers.9.attention_norm.weight                   -> blk.9.attn_norm.weight                   | BF16   | [8192]\n",
      "layers.9.ffn_norm.weight                         -> blk.9.ffn_norm.weight                    | BF16   | [8192]\n",
      "layers.10.attention.wq.weight                    -> blk.10.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.10.attention.wk.weight                    -> blk.10.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.10.attention.wv.weight                    -> blk.10.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.10.attention.wo.weight                    -> blk.10.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.10.feed_forward.w1.weight                 -> blk.10.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.10.feed_forward.w2.weight                 -> blk.10.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.10.feed_forward.w3.weight                 -> blk.10.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.10.attention_norm.weight                  -> blk.10.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.10.ffn_norm.weight                        -> blk.10.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.11.attention.wq.weight                    -> blk.11.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.11.attention.wk.weight                    -> blk.11.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.11.attention.wv.weight                    -> blk.11.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.11.attention.wo.weight                    -> blk.11.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.11.feed_forward.w1.weight                 -> blk.11.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.11.feed_forward.w2.weight                 -> blk.11.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.11.feed_forward.w3.weight                 -> blk.11.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.11.attention_norm.weight                  -> blk.11.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.11.ffn_norm.weight                        -> blk.11.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.12.attention.wq.weight                    -> blk.12.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.12.attention.wk.weight                    -> blk.12.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.12.attention.wv.weight                    -> blk.12.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.12.attention.wo.weight                    -> blk.12.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.12.feed_forward.w1.weight                 -> blk.12.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.12.feed_forward.w2.weight                 -> blk.12.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.12.feed_forward.w3.weight                 -> blk.12.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.12.attention_norm.weight                  -> blk.12.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.12.ffn_norm.weight                        -> blk.12.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.13.attention.wq.weight                    -> blk.13.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.13.attention.wk.weight                    -> blk.13.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.13.attention.wv.weight                    -> blk.13.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.13.attention.wo.weight                    -> blk.13.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.13.feed_forward.w1.weight                 -> blk.13.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.13.feed_forward.w2.weight                 -> blk.13.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.13.feed_forward.w3.weight                 -> blk.13.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.13.attention_norm.weight                  -> blk.13.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.13.ffn_norm.weight                        -> blk.13.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.14.attention.wq.weight                    -> blk.14.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.14.attention.wk.weight                    -> blk.14.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.14.attention.wv.weight                    -> blk.14.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.14.attention.wo.weight                    -> blk.14.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.14.feed_forward.w1.weight                 -> blk.14.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.14.feed_forward.w2.weight                 -> blk.14.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.14.feed_forward.w3.weight                 -> blk.14.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.14.attention_norm.weight                  -> blk.14.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.14.ffn_norm.weight                        -> blk.14.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.15.attention.wq.weight                    -> blk.15.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.15.attention.wk.weight                    -> blk.15.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.15.attention.wv.weight                    -> blk.15.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.15.attention.wo.weight                    -> blk.15.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.15.feed_forward.w1.weight                 -> blk.15.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.15.feed_forward.w2.weight                 -> blk.15.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.15.feed_forward.w3.weight                 -> blk.15.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.15.attention_norm.weight                  -> blk.15.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.15.ffn_norm.weight                        -> blk.15.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.16.attention.wq.weight                    -> blk.16.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.16.attention.wk.weight                    -> blk.16.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.16.attention.wv.weight                    -> blk.16.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.16.attention.wo.weight                    -> blk.16.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.16.feed_forward.w1.weight                 -> blk.16.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.16.feed_forward.w2.weight                 -> blk.16.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.16.feed_forward.w3.weight                 -> blk.16.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.16.attention_norm.weight                  -> blk.16.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.16.ffn_norm.weight                        -> blk.16.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.17.attention.wq.weight                    -> blk.17.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.17.attention.wk.weight                    -> blk.17.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.17.attention.wv.weight                    -> blk.17.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.17.attention.wo.weight                    -> blk.17.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.17.feed_forward.w1.weight                 -> blk.17.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.17.feed_forward.w2.weight                 -> blk.17.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.17.feed_forward.w3.weight                 -> blk.17.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.17.attention_norm.weight                  -> blk.17.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.17.ffn_norm.weight                        -> blk.17.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.18.attention.wq.weight                    -> blk.18.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.18.attention.wk.weight                    -> blk.18.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.18.attention.wv.weight                    -> blk.18.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.18.attention.wo.weight                    -> blk.18.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.18.feed_forward.w1.weight                 -> blk.18.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.18.feed_forward.w2.weight                 -> blk.18.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.18.feed_forward.w3.weight                 -> blk.18.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.18.attention_norm.weight                  -> blk.18.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.18.ffn_norm.weight                        -> blk.18.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.19.attention.wq.weight                    -> blk.19.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.19.attention.wk.weight                    -> blk.19.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.19.attention.wv.weight                    -> blk.19.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.19.attention.wo.weight                    -> blk.19.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.19.feed_forward.w1.weight                 -> blk.19.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.19.feed_forward.w2.weight                 -> blk.19.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.19.feed_forward.w3.weight                 -> blk.19.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.19.attention_norm.weight                  -> blk.19.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.19.ffn_norm.weight                        -> blk.19.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.20.attention.wq.weight                    -> blk.20.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.20.attention.wk.weight                    -> blk.20.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.20.attention.wv.weight                    -> blk.20.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.20.attention.wo.weight                    -> blk.20.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.20.feed_forward.w1.weight                 -> blk.20.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.20.feed_forward.w2.weight                 -> blk.20.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.20.feed_forward.w3.weight                 -> blk.20.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.20.attention_norm.weight                  -> blk.20.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.20.ffn_norm.weight                        -> blk.20.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.21.attention.wq.weight                    -> blk.21.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.21.attention.wk.weight                    -> blk.21.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.21.attention.wv.weight                    -> blk.21.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.21.attention.wo.weight                    -> blk.21.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.21.feed_forward.w1.weight                 -> blk.21.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.21.feed_forward.w2.weight                 -> blk.21.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.21.feed_forward.w3.weight                 -> blk.21.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.21.attention_norm.weight                  -> blk.21.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.21.ffn_norm.weight                        -> blk.21.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.22.attention.wq.weight                    -> blk.22.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.22.attention.wk.weight                    -> blk.22.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.22.attention.wv.weight                    -> blk.22.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.22.attention.wo.weight                    -> blk.22.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.22.feed_forward.w1.weight                 -> blk.22.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.22.feed_forward.w2.weight                 -> blk.22.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.22.feed_forward.w3.weight                 -> blk.22.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.22.attention_norm.weight                  -> blk.22.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.22.ffn_norm.weight                        -> blk.22.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.23.attention.wq.weight                    -> blk.23.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.23.attention.wk.weight                    -> blk.23.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.23.attention.wv.weight                    -> blk.23.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.23.attention.wo.weight                    -> blk.23.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.23.feed_forward.w1.weight                 -> blk.23.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.23.feed_forward.w2.weight                 -> blk.23.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.23.feed_forward.w3.weight                 -> blk.23.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.23.attention_norm.weight                  -> blk.23.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.23.ffn_norm.weight                        -> blk.23.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.24.attention.wq.weight                    -> blk.24.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.24.attention.wk.weight                    -> blk.24.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.24.attention.wv.weight                    -> blk.24.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.24.attention.wo.weight                    -> blk.24.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.24.feed_forward.w1.weight                 -> blk.24.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.24.feed_forward.w2.weight                 -> blk.24.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.24.feed_forward.w3.weight                 -> blk.24.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.24.attention_norm.weight                  -> blk.24.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.24.ffn_norm.weight                        -> blk.24.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.25.attention.wq.weight                    -> blk.25.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.25.attention.wk.weight                    -> blk.25.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.25.attention.wv.weight                    -> blk.25.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.25.attention.wo.weight                    -> blk.25.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.25.feed_forward.w1.weight                 -> blk.25.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.25.feed_forward.w2.weight                 -> blk.25.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.25.feed_forward.w3.weight                 -> blk.25.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.25.attention_norm.weight                  -> blk.25.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.25.ffn_norm.weight                        -> blk.25.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.26.attention.wq.weight                    -> blk.26.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.26.attention.wk.weight                    -> blk.26.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.26.attention.wv.weight                    -> blk.26.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.26.attention.wo.weight                    -> blk.26.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.26.feed_forward.w1.weight                 -> blk.26.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.26.feed_forward.w2.weight                 -> blk.26.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.26.feed_forward.w3.weight                 -> blk.26.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.26.attention_norm.weight                  -> blk.26.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.26.ffn_norm.weight                        -> blk.26.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.27.attention.wq.weight                    -> blk.27.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.27.attention.wk.weight                    -> blk.27.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.27.attention.wv.weight                    -> blk.27.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.27.attention.wo.weight                    -> blk.27.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.27.feed_forward.w1.weight                 -> blk.27.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.27.feed_forward.w2.weight                 -> blk.27.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.27.feed_forward.w3.weight                 -> blk.27.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.27.attention_norm.weight                  -> blk.27.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.27.ffn_norm.weight                        -> blk.27.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.28.attention.wq.weight                    -> blk.28.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.28.attention.wk.weight                    -> blk.28.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.28.attention.wv.weight                    -> blk.28.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.28.attention.wo.weight                    -> blk.28.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.28.feed_forward.w1.weight                 -> blk.28.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.28.feed_forward.w2.weight                 -> blk.28.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.28.feed_forward.w3.weight                 -> blk.28.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.28.attention_norm.weight                  -> blk.28.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.28.ffn_norm.weight                        -> blk.28.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.29.attention.wq.weight                    -> blk.29.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.29.attention.wk.weight                    -> blk.29.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.29.attention.wv.weight                    -> blk.29.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.29.attention.wo.weight                    -> blk.29.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.29.feed_forward.w1.weight                 -> blk.29.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.29.feed_forward.w2.weight                 -> blk.29.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.29.feed_forward.w3.weight                 -> blk.29.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.29.attention_norm.weight                  -> blk.29.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.29.ffn_norm.weight                        -> blk.29.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.30.attention.wq.weight                    -> blk.30.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.30.attention.wk.weight                    -> blk.30.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.30.attention.wv.weight                    -> blk.30.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.30.attention.wo.weight                    -> blk.30.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.30.feed_forward.w1.weight                 -> blk.30.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.30.feed_forward.w2.weight                 -> blk.30.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.30.feed_forward.w3.weight                 -> blk.30.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.30.attention_norm.weight                  -> blk.30.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.30.ffn_norm.weight                        -> blk.30.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.31.attention.wq.weight                    -> blk.31.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.31.attention.wk.weight                    -> blk.31.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.31.attention.wv.weight                    -> blk.31.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.31.attention.wo.weight                    -> blk.31.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.31.feed_forward.w1.weight                 -> blk.31.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.31.feed_forward.w2.weight                 -> blk.31.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.31.feed_forward.w3.weight                 -> blk.31.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.31.attention_norm.weight                  -> blk.31.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.31.ffn_norm.weight                        -> blk.31.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.32.attention.wq.weight                    -> blk.32.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.32.attention.wk.weight                    -> blk.32.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.32.attention.wv.weight                    -> blk.32.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.32.attention.wo.weight                    -> blk.32.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.32.feed_forward.w1.weight                 -> blk.32.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.32.feed_forward.w2.weight                 -> blk.32.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.32.feed_forward.w3.weight                 -> blk.32.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.32.attention_norm.weight                  -> blk.32.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.32.ffn_norm.weight                        -> blk.32.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.33.attention.wq.weight                    -> blk.33.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.33.attention.wk.weight                    -> blk.33.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.33.attention.wv.weight                    -> blk.33.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.33.attention.wo.weight                    -> blk.33.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.33.feed_forward.w1.weight                 -> blk.33.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.33.feed_forward.w2.weight                 -> blk.33.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.33.feed_forward.w3.weight                 -> blk.33.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.33.attention_norm.weight                  -> blk.33.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.33.ffn_norm.weight                        -> blk.33.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.34.attention.wq.weight                    -> blk.34.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.34.attention.wk.weight                    -> blk.34.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.34.attention.wv.weight                    -> blk.34.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.34.attention.wo.weight                    -> blk.34.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.34.feed_forward.w1.weight                 -> blk.34.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.34.feed_forward.w2.weight                 -> blk.34.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.34.feed_forward.w3.weight                 -> blk.34.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.34.attention_norm.weight                  -> blk.34.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.34.ffn_norm.weight                        -> blk.34.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.35.attention.wq.weight                    -> blk.35.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.35.attention.wk.weight                    -> blk.35.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.35.attention.wv.weight                    -> blk.35.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.35.attention.wo.weight                    -> blk.35.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.35.feed_forward.w1.weight                 -> blk.35.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.35.feed_forward.w2.weight                 -> blk.35.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.35.feed_forward.w3.weight                 -> blk.35.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.35.attention_norm.weight                  -> blk.35.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.35.ffn_norm.weight                        -> blk.35.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.36.attention.wq.weight                    -> blk.36.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.36.attention.wk.weight                    -> blk.36.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.36.attention.wv.weight                    -> blk.36.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.36.attention.wo.weight                    -> blk.36.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.36.feed_forward.w1.weight                 -> blk.36.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.36.feed_forward.w2.weight                 -> blk.36.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.36.feed_forward.w3.weight                 -> blk.36.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.36.attention_norm.weight                  -> blk.36.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.36.ffn_norm.weight                        -> blk.36.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.37.attention.wq.weight                    -> blk.37.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.37.attention.wk.weight                    -> blk.37.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.37.attention.wv.weight                    -> blk.37.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.37.attention.wo.weight                    -> blk.37.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.37.feed_forward.w1.weight                 -> blk.37.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.37.feed_forward.w2.weight                 -> blk.37.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.37.feed_forward.w3.weight                 -> blk.37.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.37.attention_norm.weight                  -> blk.37.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.37.ffn_norm.weight                        -> blk.37.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.38.attention.wq.weight                    -> blk.38.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.38.attention.wk.weight                    -> blk.38.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.38.attention.wv.weight                    -> blk.38.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.38.attention.wo.weight                    -> blk.38.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.38.feed_forward.w1.weight                 -> blk.38.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.38.feed_forward.w2.weight                 -> blk.38.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.38.feed_forward.w3.weight                 -> blk.38.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.38.attention_norm.weight                  -> blk.38.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.38.ffn_norm.weight                        -> blk.38.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.39.attention.wq.weight                    -> blk.39.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.39.attention.wk.weight                    -> blk.39.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.39.attention.wv.weight                    -> blk.39.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.39.attention.wo.weight                    -> blk.39.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.39.feed_forward.w1.weight                 -> blk.39.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.39.feed_forward.w2.weight                 -> blk.39.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.39.feed_forward.w3.weight                 -> blk.39.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.39.attention_norm.weight                  -> blk.39.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.39.ffn_norm.weight                        -> blk.39.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.40.attention.wq.weight                    -> blk.40.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.40.attention.wk.weight                    -> blk.40.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.40.attention.wv.weight                    -> blk.40.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.40.attention.wo.weight                    -> blk.40.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.40.feed_forward.w1.weight                 -> blk.40.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.40.feed_forward.w2.weight                 -> blk.40.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.40.feed_forward.w3.weight                 -> blk.40.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.40.attention_norm.weight                  -> blk.40.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.40.ffn_norm.weight                        -> blk.40.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.41.attention.wq.weight                    -> blk.41.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.41.attention.wk.weight                    -> blk.41.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.41.attention.wv.weight                    -> blk.41.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.41.attention.wo.weight                    -> blk.41.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.41.feed_forward.w1.weight                 -> blk.41.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.41.feed_forward.w2.weight                 -> blk.41.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.41.feed_forward.w3.weight                 -> blk.41.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.41.attention_norm.weight                  -> blk.41.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.41.ffn_norm.weight                        -> blk.41.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.42.attention.wq.weight                    -> blk.42.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.42.attention.wk.weight                    -> blk.42.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.42.attention.wv.weight                    -> blk.42.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.42.attention.wo.weight                    -> blk.42.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.42.feed_forward.w1.weight                 -> blk.42.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.42.feed_forward.w2.weight                 -> blk.42.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.42.feed_forward.w3.weight                 -> blk.42.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.42.attention_norm.weight                  -> blk.42.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.42.ffn_norm.weight                        -> blk.42.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.43.attention.wq.weight                    -> blk.43.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.43.attention.wk.weight                    -> blk.43.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.43.attention.wv.weight                    -> blk.43.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.43.attention.wo.weight                    -> blk.43.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.43.feed_forward.w1.weight                 -> blk.43.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.43.feed_forward.w2.weight                 -> blk.43.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.43.feed_forward.w3.weight                 -> blk.43.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.43.attention_norm.weight                  -> blk.43.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.43.ffn_norm.weight                        -> blk.43.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.44.attention.wq.weight                    -> blk.44.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.44.attention.wk.weight                    -> blk.44.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.44.attention.wv.weight                    -> blk.44.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.44.attention.wo.weight                    -> blk.44.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.44.feed_forward.w1.weight                 -> blk.44.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.44.feed_forward.w2.weight                 -> blk.44.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.44.feed_forward.w3.weight                 -> blk.44.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.44.attention_norm.weight                  -> blk.44.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.44.ffn_norm.weight                        -> blk.44.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.45.attention.wq.weight                    -> blk.45.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.45.attention.wk.weight                    -> blk.45.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.45.attention.wv.weight                    -> blk.45.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.45.attention.wo.weight                    -> blk.45.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.45.feed_forward.w1.weight                 -> blk.45.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.45.feed_forward.w2.weight                 -> blk.45.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.45.feed_forward.w3.weight                 -> blk.45.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.45.attention_norm.weight                  -> blk.45.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.45.ffn_norm.weight                        -> blk.45.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.46.attention.wq.weight                    -> blk.46.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.46.attention.wk.weight                    -> blk.46.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.46.attention.wv.weight                    -> blk.46.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.46.attention.wo.weight                    -> blk.46.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.46.feed_forward.w1.weight                 -> blk.46.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.46.feed_forward.w2.weight                 -> blk.46.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.46.feed_forward.w3.weight                 -> blk.46.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.46.attention_norm.weight                  -> blk.46.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.46.ffn_norm.weight                        -> blk.46.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.47.attention.wq.weight                    -> blk.47.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.47.attention.wk.weight                    -> blk.47.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.47.attention.wv.weight                    -> blk.47.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.47.attention.wo.weight                    -> blk.47.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.47.feed_forward.w1.weight                 -> blk.47.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.47.feed_forward.w2.weight                 -> blk.47.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.47.feed_forward.w3.weight                 -> blk.47.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.47.attention_norm.weight                  -> blk.47.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.47.ffn_norm.weight                        -> blk.47.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.48.attention.wq.weight                    -> blk.48.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.48.attention.wk.weight                    -> blk.48.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.48.attention.wv.weight                    -> blk.48.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.48.attention.wo.weight                    -> blk.48.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.48.feed_forward.w1.weight                 -> blk.48.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.48.feed_forward.w2.weight                 -> blk.48.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.48.feed_forward.w3.weight                 -> blk.48.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.48.attention_norm.weight                  -> blk.48.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.48.ffn_norm.weight                        -> blk.48.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.49.attention.wq.weight                    -> blk.49.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.49.attention.wk.weight                    -> blk.49.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.49.attention.wv.weight                    -> blk.49.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.49.attention.wo.weight                    -> blk.49.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.49.feed_forward.w1.weight                 -> blk.49.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.49.feed_forward.w2.weight                 -> blk.49.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.49.feed_forward.w3.weight                 -> blk.49.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.49.attention_norm.weight                  -> blk.49.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.49.ffn_norm.weight                        -> blk.49.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.50.attention.wq.weight                    -> blk.50.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.50.attention.wk.weight                    -> blk.50.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.50.attention.wv.weight                    -> blk.50.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.50.attention.wo.weight                    -> blk.50.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.50.feed_forward.w1.weight                 -> blk.50.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.50.feed_forward.w2.weight                 -> blk.50.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.50.feed_forward.w3.weight                 -> blk.50.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.50.attention_norm.weight                  -> blk.50.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.50.ffn_norm.weight                        -> blk.50.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.51.attention.wq.weight                    -> blk.51.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.51.attention.wk.weight                    -> blk.51.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.51.attention.wv.weight                    -> blk.51.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.51.attention.wo.weight                    -> blk.51.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.51.feed_forward.w1.weight                 -> blk.51.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.51.feed_forward.w2.weight                 -> blk.51.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.51.feed_forward.w3.weight                 -> blk.51.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.51.attention_norm.weight                  -> blk.51.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.51.ffn_norm.weight                        -> blk.51.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.52.attention.wq.weight                    -> blk.52.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.52.attention.wk.weight                    -> blk.52.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.52.attention.wv.weight                    -> blk.52.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.52.attention.wo.weight                    -> blk.52.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.52.feed_forward.w1.weight                 -> blk.52.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.52.feed_forward.w2.weight                 -> blk.52.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.52.feed_forward.w3.weight                 -> blk.52.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.52.attention_norm.weight                  -> blk.52.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.52.ffn_norm.weight                        -> blk.52.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.53.attention.wq.weight                    -> blk.53.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.53.attention.wk.weight                    -> blk.53.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.53.attention.wv.weight                    -> blk.53.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.53.attention.wo.weight                    -> blk.53.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.53.feed_forward.w1.weight                 -> blk.53.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.53.feed_forward.w2.weight                 -> blk.53.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.53.feed_forward.w3.weight                 -> blk.53.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.53.attention_norm.weight                  -> blk.53.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.53.ffn_norm.weight                        -> blk.53.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.54.attention.wq.weight                    -> blk.54.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.54.attention.wk.weight                    -> blk.54.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.54.attention.wv.weight                    -> blk.54.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.54.attention.wo.weight                    -> blk.54.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.54.feed_forward.w1.weight                 -> blk.54.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.54.feed_forward.w2.weight                 -> blk.54.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.54.feed_forward.w3.weight                 -> blk.54.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.54.attention_norm.weight                  -> blk.54.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.54.ffn_norm.weight                        -> blk.54.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.55.attention.wq.weight                    -> blk.55.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.55.attention.wk.weight                    -> blk.55.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.55.attention.wv.weight                    -> blk.55.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.55.attention.wo.weight                    -> blk.55.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.55.feed_forward.w1.weight                 -> blk.55.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.55.feed_forward.w2.weight                 -> blk.55.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.55.feed_forward.w3.weight                 -> blk.55.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.55.attention_norm.weight                  -> blk.55.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.55.ffn_norm.weight                        -> blk.55.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.56.attention.wq.weight                    -> blk.56.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.56.attention.wk.weight                    -> blk.56.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.56.attention.wv.weight                    -> blk.56.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.56.attention.wo.weight                    -> blk.56.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.56.feed_forward.w1.weight                 -> blk.56.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.56.feed_forward.w2.weight                 -> blk.56.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.56.feed_forward.w3.weight                 -> blk.56.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.56.attention_norm.weight                  -> blk.56.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.56.ffn_norm.weight                        -> blk.56.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.57.attention.wq.weight                    -> blk.57.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.57.attention.wk.weight                    -> blk.57.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.57.attention.wv.weight                    -> blk.57.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.57.attention.wo.weight                    -> blk.57.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.57.feed_forward.w1.weight                 -> blk.57.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.57.feed_forward.w2.weight                 -> blk.57.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.57.feed_forward.w3.weight                 -> blk.57.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.57.attention_norm.weight                  -> blk.57.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.57.ffn_norm.weight                        -> blk.57.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.58.attention.wq.weight                    -> blk.58.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.58.attention.wk.weight                    -> blk.58.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.58.attention.wv.weight                    -> blk.58.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.58.attention.wo.weight                    -> blk.58.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.58.feed_forward.w1.weight                 -> blk.58.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.58.feed_forward.w2.weight                 -> blk.58.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.58.feed_forward.w3.weight                 -> blk.58.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.58.attention_norm.weight                  -> blk.58.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.58.ffn_norm.weight                        -> blk.58.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.59.attention.wq.weight                    -> blk.59.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.59.attention.wk.weight                    -> blk.59.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.59.attention.wv.weight                    -> blk.59.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.59.attention.wo.weight                    -> blk.59.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.59.feed_forward.w1.weight                 -> blk.59.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.59.feed_forward.w2.weight                 -> blk.59.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.59.feed_forward.w3.weight                 -> blk.59.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.59.attention_norm.weight                  -> blk.59.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.59.ffn_norm.weight                        -> blk.59.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.60.attention.wq.weight                    -> blk.60.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.60.attention.wk.weight                    -> blk.60.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.60.attention.wv.weight                    -> blk.60.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.60.attention.wo.weight                    -> blk.60.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.60.feed_forward.w1.weight                 -> blk.60.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.60.feed_forward.w2.weight                 -> blk.60.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.60.feed_forward.w3.weight                 -> blk.60.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.60.attention_norm.weight                  -> blk.60.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.60.ffn_norm.weight                        -> blk.60.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.61.attention.wq.weight                    -> blk.61.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.61.attention.wk.weight                    -> blk.61.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.61.attention.wv.weight                    -> blk.61.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.61.attention.wo.weight                    -> blk.61.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.61.feed_forward.w1.weight                 -> blk.61.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.61.feed_forward.w2.weight                 -> blk.61.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.61.feed_forward.w3.weight                 -> blk.61.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.61.attention_norm.weight                  -> blk.61.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.61.ffn_norm.weight                        -> blk.61.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.62.attention.wq.weight                    -> blk.62.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.62.attention.wk.weight                    -> blk.62.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.62.attention.wv.weight                    -> blk.62.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.62.attention.wo.weight                    -> blk.62.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.62.feed_forward.w1.weight                 -> blk.62.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.62.feed_forward.w2.weight                 -> blk.62.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.62.feed_forward.w3.weight                 -> blk.62.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.62.attention_norm.weight                  -> blk.62.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.62.ffn_norm.weight                        -> blk.62.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.63.attention.wq.weight                    -> blk.63.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.63.attention.wk.weight                    -> blk.63.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.63.attention.wv.weight                    -> blk.63.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.63.attention.wo.weight                    -> blk.63.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.63.feed_forward.w1.weight                 -> blk.63.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.63.feed_forward.w2.weight                 -> blk.63.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.63.feed_forward.w3.weight                 -> blk.63.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.63.attention_norm.weight                  -> blk.63.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.63.ffn_norm.weight                        -> blk.63.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.64.attention.wq.weight                    -> blk.64.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.64.attention.wk.weight                    -> blk.64.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.64.attention.wv.weight                    -> blk.64.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.64.attention.wo.weight                    -> blk.64.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.64.feed_forward.w1.weight                 -> blk.64.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.64.feed_forward.w2.weight                 -> blk.64.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.64.feed_forward.w3.weight                 -> blk.64.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.64.attention_norm.weight                  -> blk.64.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.64.ffn_norm.weight                        -> blk.64.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.65.attention.wq.weight                    -> blk.65.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.65.attention.wk.weight                    -> blk.65.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.65.attention.wv.weight                    -> blk.65.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.65.attention.wo.weight                    -> blk.65.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.65.feed_forward.w1.weight                 -> blk.65.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.65.feed_forward.w2.weight                 -> blk.65.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.65.feed_forward.w3.weight                 -> blk.65.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.65.attention_norm.weight                  -> blk.65.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.65.ffn_norm.weight                        -> blk.65.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.66.attention.wq.weight                    -> blk.66.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.66.attention.wk.weight                    -> blk.66.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.66.attention.wv.weight                    -> blk.66.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.66.attention.wo.weight                    -> blk.66.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.66.feed_forward.w1.weight                 -> blk.66.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.66.feed_forward.w2.weight                 -> blk.66.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.66.feed_forward.w3.weight                 -> blk.66.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.66.attention_norm.weight                  -> blk.66.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.66.ffn_norm.weight                        -> blk.66.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.67.attention.wq.weight                    -> blk.67.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.67.attention.wk.weight                    -> blk.67.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.67.attention.wv.weight                    -> blk.67.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.67.attention.wo.weight                    -> blk.67.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.67.feed_forward.w1.weight                 -> blk.67.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.67.feed_forward.w2.weight                 -> blk.67.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.67.feed_forward.w3.weight                 -> blk.67.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.67.attention_norm.weight                  -> blk.67.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.67.ffn_norm.weight                        -> blk.67.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.68.attention.wq.weight                    -> blk.68.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.68.attention.wk.weight                    -> blk.68.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.68.attention.wv.weight                    -> blk.68.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.68.attention.wo.weight                    -> blk.68.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.68.feed_forward.w1.weight                 -> blk.68.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.68.feed_forward.w2.weight                 -> blk.68.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.68.feed_forward.w3.weight                 -> blk.68.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.68.attention_norm.weight                  -> blk.68.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.68.ffn_norm.weight                        -> blk.68.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.69.attention.wq.weight                    -> blk.69.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.69.attention.wk.weight                    -> blk.69.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.69.attention.wv.weight                    -> blk.69.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.69.attention.wo.weight                    -> blk.69.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.69.feed_forward.w1.weight                 -> blk.69.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.69.feed_forward.w2.weight                 -> blk.69.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.69.feed_forward.w3.weight                 -> blk.69.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.69.attention_norm.weight                  -> blk.69.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.69.ffn_norm.weight                        -> blk.69.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.70.attention.wq.weight                    -> blk.70.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.70.attention.wk.weight                    -> blk.70.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.70.attention.wv.weight                    -> blk.70.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.70.attention.wo.weight                    -> blk.70.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.70.feed_forward.w1.weight                 -> blk.70.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.70.feed_forward.w2.weight                 -> blk.70.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.70.feed_forward.w3.weight                 -> blk.70.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.70.attention_norm.weight                  -> blk.70.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.70.ffn_norm.weight                        -> blk.70.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.71.attention.wq.weight                    -> blk.71.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.71.attention.wk.weight                    -> blk.71.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.71.attention.wv.weight                    -> blk.71.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.71.attention.wo.weight                    -> blk.71.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.71.feed_forward.w1.weight                 -> blk.71.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.71.feed_forward.w2.weight                 -> blk.71.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.71.feed_forward.w3.weight                 -> blk.71.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.71.attention_norm.weight                  -> blk.71.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.71.ffn_norm.weight                        -> blk.71.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.72.attention.wq.weight                    -> blk.72.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.72.attention.wk.weight                    -> blk.72.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.72.attention.wv.weight                    -> blk.72.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.72.attention.wo.weight                    -> blk.72.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.72.feed_forward.w1.weight                 -> blk.72.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.72.feed_forward.w2.weight                 -> blk.72.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.72.feed_forward.w3.weight                 -> blk.72.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.72.attention_norm.weight                  -> blk.72.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.72.ffn_norm.weight                        -> blk.72.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.73.attention.wq.weight                    -> blk.73.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.73.attention.wk.weight                    -> blk.73.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.73.attention.wv.weight                    -> blk.73.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.73.attention.wo.weight                    -> blk.73.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.73.feed_forward.w1.weight                 -> blk.73.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.73.feed_forward.w2.weight                 -> blk.73.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.73.feed_forward.w3.weight                 -> blk.73.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.73.attention_norm.weight                  -> blk.73.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.73.ffn_norm.weight                        -> blk.73.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.74.attention.wq.weight                    -> blk.74.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.74.attention.wk.weight                    -> blk.74.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.74.attention.wv.weight                    -> blk.74.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.74.attention.wo.weight                    -> blk.74.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.74.feed_forward.w1.weight                 -> blk.74.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.74.feed_forward.w2.weight                 -> blk.74.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.74.feed_forward.w3.weight                 -> blk.74.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.74.attention_norm.weight                  -> blk.74.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.74.ffn_norm.weight                        -> blk.74.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.75.attention.wq.weight                    -> blk.75.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.75.attention.wk.weight                    -> blk.75.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.75.attention.wv.weight                    -> blk.75.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.75.attention.wo.weight                    -> blk.75.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.75.feed_forward.w1.weight                 -> blk.75.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.75.feed_forward.w2.weight                 -> blk.75.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.75.feed_forward.w3.weight                 -> blk.75.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.75.attention_norm.weight                  -> blk.75.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.75.ffn_norm.weight                        -> blk.75.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.76.attention.wq.weight                    -> blk.76.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.76.attention.wk.weight                    -> blk.76.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.76.attention.wv.weight                    -> blk.76.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.76.attention.wo.weight                    -> blk.76.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.76.feed_forward.w1.weight                 -> blk.76.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.76.feed_forward.w2.weight                 -> blk.76.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.76.feed_forward.w3.weight                 -> blk.76.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.76.attention_norm.weight                  -> blk.76.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.76.ffn_norm.weight                        -> blk.76.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.77.attention.wq.weight                    -> blk.77.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.77.attention.wk.weight                    -> blk.77.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.77.attention.wv.weight                    -> blk.77.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.77.attention.wo.weight                    -> blk.77.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.77.feed_forward.w1.weight                 -> blk.77.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.77.feed_forward.w2.weight                 -> blk.77.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.77.feed_forward.w3.weight                 -> blk.77.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.77.attention_norm.weight                  -> blk.77.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.77.ffn_norm.weight                        -> blk.77.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.78.attention.wq.weight                    -> blk.78.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.78.attention.wk.weight                    -> blk.78.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.78.attention.wv.weight                    -> blk.78.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.78.attention.wo.weight                    -> blk.78.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.78.feed_forward.w1.weight                 -> blk.78.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.78.feed_forward.w2.weight                 -> blk.78.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.78.feed_forward.w3.weight                 -> blk.78.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.78.attention_norm.weight                  -> blk.78.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.78.ffn_norm.weight                        -> blk.78.ffn_norm.weight                   | BF16   | [8192]\n",
      "layers.79.attention.wq.weight                    -> blk.79.attn_q.weight                     | BF16   | [8192, 8192]\n",
      "layers.79.attention.wk.weight                    -> blk.79.attn_k.weight                     | BF16   | [1024, 8192]\n",
      "layers.79.attention.wv.weight                    -> blk.79.attn_v.weight                     | BF16   | [1024, 8192]\n",
      "layers.79.attention.wo.weight                    -> blk.79.attn_output.weight                | BF16   | [8192, 8192]\n",
      "layers.79.feed_forward.w1.weight                 -> blk.79.ffn_gate.weight                   | BF16   | [28672, 8192]\n",
      "layers.79.feed_forward.w2.weight                 -> blk.79.ffn_down.weight                   | BF16   | [8192, 28672]\n",
      "layers.79.feed_forward.w3.weight                 -> blk.79.ffn_up.weight                     | BF16   | [28672, 8192]\n",
      "layers.79.attention_norm.weight                  -> blk.79.attn_norm.weight                  | BF16   | [8192]\n",
      "layers.79.ffn_norm.weight                        -> blk.79.ffn_norm.weight                   | BF16   | [8192]\n",
      "skipping tensor rope_freqs\n",
      "Writing models/70B-v2/ggml-model-f16.gguf, format 1\n",
      "gguf: This GGUF file is for Little Endian only\n",
      "gguf: Adding 61249 merge(s).\n",
      "gguf: Setting special token type bos to 1\n",
      "gguf: Setting special token type eos to 2\n",
      "gguf: Setting special token type unk to 0\n",
      "gguf: Setting add_bos_token to True\n",
      "gguf: Setting add_eos_token to False\n",
      "[  1/723] Writing tensor token_embd.weight                      | size  32000 x   8192  | type F16  | T+   2\n",
      "[  2/723] Writing tensor output_norm.weight                     | size   8192           | type F32  | T+   2\n",
      "[  3/723] Writing tensor output.weight                          | size  32000 x   8192  | type F16  | T+   2\n",
      "[  4/723] Writing tensor blk.0.attn_q.weight                    | size   8192 x   8192  | type F16  | T+   2\n",
      "[  5/723] Writing tensor blk.0.attn_k.weight                    | size   1024 x   8192  | type F16  | T+   2\n",
      "[  6/723] Writing tensor blk.0.attn_v.weight                    | size   1024 x   8192  | type F16  | T+   2\n",
      "[  7/723] Writing tensor blk.0.attn_output.weight               | size   8192 x   8192  | type F16  | T+   2\n",
      "[  8/723] Writing tensor blk.0.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+   2\n",
      "[  9/723] Writing tensor blk.0.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+   3\n",
      "[ 10/723] Writing tensor blk.0.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+   4\n",
      "[ 11/723] Writing tensor blk.0.attn_norm.weight                 | size   8192           | type F32  | T+   4\n",
      "[ 12/723] Writing tensor blk.0.ffn_norm.weight                  | size   8192           | type F32  | T+   4\n",
      "[ 13/723] Writing tensor blk.1.attn_q.weight                    | size   8192 x   8192  | type F16  | T+   4\n",
      "[ 14/723] Writing tensor blk.1.attn_k.weight                    | size   1024 x   8192  | type F16  | T+   4\n",
      "[ 15/723] Writing tensor blk.1.attn_v.weight                    | size   1024 x   8192  | type F16  | T+   4\n",
      "[ 16/723] Writing tensor blk.1.attn_output.weight               | size   8192 x   8192  | type F16  | T+   4\n",
      "[ 17/723] Writing tensor blk.1.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+   5\n",
      "[ 18/723] Writing tensor blk.1.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+   5\n",
      "[ 19/723] Writing tensor blk.1.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+   6\n",
      "[ 20/723] Writing tensor blk.1.attn_norm.weight                 | size   8192           | type F32  | T+   6\n",
      "[ 21/723] Writing tensor blk.1.ffn_norm.weight                  | size   8192           | type F32  | T+   6\n",
      "[ 22/723] Writing tensor blk.2.attn_q.weight                    | size   8192 x   8192  | type F16  | T+   6\n",
      "[ 23/723] Writing tensor blk.2.attn_k.weight                    | size   1024 x   8192  | type F16  | T+   6\n",
      "[ 24/723] Writing tensor blk.2.attn_v.weight                    | size   1024 x   8192  | type F16  | T+   6\n",
      "[ 25/723] Writing tensor blk.2.attn_output.weight               | size   8192 x   8192  | type F16  | T+   6\n",
      "[ 26/723] Writing tensor blk.2.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+   7\n",
      "[ 27/723] Writing tensor blk.2.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+   7\n",
      "[ 28/723] Writing tensor blk.2.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+   8\n",
      "[ 29/723] Writing tensor blk.2.attn_norm.weight                 | size   8192           | type F32  | T+   8\n",
      "[ 30/723] Writing tensor blk.2.ffn_norm.weight                  | size   8192           | type F32  | T+   8\n",
      "[ 31/723] Writing tensor blk.3.attn_q.weight                    | size   8192 x   8192  | type F16  | T+   8\n",
      "[ 32/723] Writing tensor blk.3.attn_k.weight                    | size   1024 x   8192  | type F16  | T+   8\n",
      "[ 33/723] Writing tensor blk.3.attn_v.weight                    | size   1024 x   8192  | type F16  | T+   8\n",
      "[ 34/723] Writing tensor blk.3.attn_output.weight               | size   8192 x   8192  | type F16  | T+   8\n",
      "[ 35/723] Writing tensor blk.3.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+   9\n",
      "[ 36/723] Writing tensor blk.3.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+   9\n",
      "[ 37/723] Writing tensor blk.3.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+  10\n",
      "[ 38/723] Writing tensor blk.3.attn_norm.weight                 | size   8192           | type F32  | T+  10\n",
      "[ 39/723] Writing tensor blk.3.ffn_norm.weight                  | size   8192           | type F32  | T+  10\n",
      "[ 40/723] Writing tensor blk.4.attn_q.weight                    | size   8192 x   8192  | type F16  | T+  10\n",
      "[ 41/723] Writing tensor blk.4.attn_k.weight                    | size   1024 x   8192  | type F16  | T+  10\n",
      "[ 42/723] Writing tensor blk.4.attn_v.weight                    | size   1024 x   8192  | type F16  | T+  10\n",
      "[ 43/723] Writing tensor blk.4.attn_output.weight               | size   8192 x   8192  | type F16  | T+  10\n",
      "[ 44/723] Writing tensor blk.4.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+  11\n",
      "[ 45/723] Writing tensor blk.4.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+  12\n",
      "[ 46/723] Writing tensor blk.4.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+  12\n",
      "[ 47/723] Writing tensor blk.4.attn_norm.weight                 | size   8192           | type F32  | T+  12\n",
      "[ 48/723] Writing tensor blk.4.ffn_norm.weight                  | size   8192           | type F32  | T+  12\n",
      "[ 49/723] Writing tensor blk.5.attn_q.weight                    | size   8192 x   8192  | type F16  | T+  12\n",
      "[ 50/723] Writing tensor blk.5.attn_k.weight                    | size   1024 x   8192  | type F16  | T+  12\n",
      "[ 51/723] Writing tensor blk.5.attn_v.weight                    | size   1024 x   8192  | type F16  | T+  12\n",
      "[ 52/723] Writing tensor blk.5.attn_output.weight               | size   8192 x   8192  | type F16  | T+  12\n",
      "[ 53/723] Writing tensor blk.5.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+  13\n",
      "[ 54/723] Writing tensor blk.5.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+  14\n",
      "[ 55/723] Writing tensor blk.5.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+  14\n",
      "[ 56/723] Writing tensor blk.5.attn_norm.weight                 | size   8192           | type F32  | T+  14\n",
      "[ 57/723] Writing tensor blk.5.ffn_norm.weight                  | size   8192           | type F32  | T+  14\n",
      "[ 58/723] Writing tensor blk.6.attn_q.weight                    | size   8192 x   8192  | type F16  | T+  14\n",
      "[ 59/723] Writing tensor blk.6.attn_k.weight                    | size   1024 x   8192  | type F16  | T+  14\n",
      "[ 60/723] Writing tensor blk.6.attn_v.weight                    | size   1024 x   8192  | type F16  | T+  14\n",
      "[ 61/723] Writing tensor blk.6.attn_output.weight               | size   8192 x   8192  | type F16  | T+  14\n",
      "[ 62/723] Writing tensor blk.6.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+  15\n",
      "[ 63/723] Writing tensor blk.6.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+  16\n",
      "[ 64/723] Writing tensor blk.6.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+  16\n",
      "[ 65/723] Writing tensor blk.6.attn_norm.weight                 | size   8192           | type F32  | T+  16\n",
      "[ 66/723] Writing tensor blk.6.ffn_norm.weight                  | size   8192           | type F32  | T+  16\n",
      "[ 67/723] Writing tensor blk.7.attn_q.weight                    | size   8192 x   8192  | type F16  | T+  16\n",
      "[ 68/723] Writing tensor blk.7.attn_k.weight                    | size   1024 x   8192  | type F16  | T+  16\n",
      "[ 69/723] Writing tensor blk.7.attn_v.weight                    | size   1024 x   8192  | type F16  | T+  16\n",
      "[ 70/723] Writing tensor blk.7.attn_output.weight               | size   8192 x   8192  | type F16  | T+  16\n",
      "[ 71/723] Writing tensor blk.7.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+  18\n",
      "[ 72/723] Writing tensor blk.7.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+  18\n",
      "[ 73/723] Writing tensor blk.7.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+  18\n",
      "[ 74/723] Writing tensor blk.7.attn_norm.weight                 | size   8192           | type F32  | T+  18\n",
      "[ 75/723] Writing tensor blk.7.ffn_norm.weight                  | size   8192           | type F32  | T+  18\n",
      "[ 76/723] Writing tensor blk.8.attn_q.weight                    | size   8192 x   8192  | type F16  | T+  18\n",
      "[ 77/723] Writing tensor blk.8.attn_k.weight                    | size   1024 x   8192  | type F16  | T+  18\n",
      "[ 78/723] Writing tensor blk.8.attn_v.weight                    | size   1024 x   8192  | type F16  | T+  18\n",
      "[ 79/723] Writing tensor blk.8.attn_output.weight               | size   8192 x   8192  | type F16  | T+  18\n",
      "[ 80/723] Writing tensor blk.8.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+  20\n",
      "[ 81/723] Writing tensor blk.8.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+  20\n",
      "[ 82/723] Writing tensor blk.8.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+  20\n",
      "[ 83/723] Writing tensor blk.8.attn_norm.weight                 | size   8192           | type F32  | T+  20\n",
      "[ 84/723] Writing tensor blk.8.ffn_norm.weight                  | size   8192           | type F32  | T+  20\n",
      "[ 85/723] Writing tensor blk.9.attn_q.weight                    | size   8192 x   8192  | type F16  | T+  20\n",
      "[ 86/723] Writing tensor blk.9.attn_k.weight                    | size   1024 x   8192  | type F16  | T+  20\n",
      "[ 87/723] Writing tensor blk.9.attn_v.weight                    | size   1024 x   8192  | type F16  | T+  20\n",
      "[ 88/723] Writing tensor blk.9.attn_output.weight               | size   8192 x   8192  | type F16  | T+  20\n",
      "[ 89/723] Writing tensor blk.9.ffn_gate.weight                  | size  28672 x   8192  | type F16  | T+  22\n",
      "[ 90/723] Writing tensor blk.9.ffn_down.weight                  | size   8192 x  28672  | type F16  | T+  22\n",
      "[ 91/723] Writing tensor blk.9.ffn_up.weight                    | size  28672 x   8192  | type F16  | T+  22\n",
      "[ 92/723] Writing tensor blk.9.attn_norm.weight                 | size   8192           | type F32  | T+  22\n",
      "[ 93/723] Writing tensor blk.9.ffn_norm.weight                  | size   8192           | type F32  | T+  22\n",
      "[ 94/723] Writing tensor blk.10.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  22\n",
      "[ 95/723] Writing tensor blk.10.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  22\n",
      "[ 96/723] Writing tensor blk.10.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  22\n",
      "[ 97/723] Writing tensor blk.10.attn_output.weight              | size   8192 x   8192  | type F16  | T+  22\n",
      "[ 98/723] Writing tensor blk.10.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  24\n",
      "[ 99/723] Writing tensor blk.10.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  24\n",
      "[100/723] Writing tensor blk.10.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  24\n",
      "[101/723] Writing tensor blk.10.attn_norm.weight                | size   8192           | type F32  | T+  24\n",
      "[102/723] Writing tensor blk.10.ffn_norm.weight                 | size   8192           | type F32  | T+  24\n",
      "[103/723] Writing tensor blk.11.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  24\n",
      "[104/723] Writing tensor blk.11.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  24\n",
      "[105/723] Writing tensor blk.11.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  24\n",
      "[106/723] Writing tensor blk.11.attn_output.weight              | size   8192 x   8192  | type F16  | T+  24\n",
      "[107/723] Writing tensor blk.11.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  26\n",
      "[108/723] Writing tensor blk.11.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  26\n",
      "[109/723] Writing tensor blk.11.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  26\n",
      "[110/723] Writing tensor blk.11.attn_norm.weight                | size   8192           | type F32  | T+  26\n",
      "[111/723] Writing tensor blk.11.ffn_norm.weight                 | size   8192           | type F32  | T+  26\n",
      "[112/723] Writing tensor blk.12.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  26\n",
      "[113/723] Writing tensor blk.12.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  26\n",
      "[114/723] Writing tensor blk.12.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  26\n",
      "[115/723] Writing tensor blk.12.attn_output.weight              | size   8192 x   8192  | type F16  | T+  26\n",
      "[116/723] Writing tensor blk.12.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  28\n",
      "[117/723] Writing tensor blk.12.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  28\n",
      "[118/723] Writing tensor blk.12.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  28\n",
      "[119/723] Writing tensor blk.12.attn_norm.weight                | size   8192           | type F32  | T+  28\n",
      "[120/723] Writing tensor blk.12.ffn_norm.weight                 | size   8192           | type F32  | T+  28\n",
      "[121/723] Writing tensor blk.13.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  28\n",
      "[122/723] Writing tensor blk.13.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  28\n",
      "[123/723] Writing tensor blk.13.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  28\n",
      "[124/723] Writing tensor blk.13.attn_output.weight              | size   8192 x   8192  | type F16  | T+  28\n",
      "[125/723] Writing tensor blk.13.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  30\n",
      "[126/723] Writing tensor blk.13.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  30\n",
      "[127/723] Writing tensor blk.13.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  31\n",
      "[128/723] Writing tensor blk.13.attn_norm.weight                | size   8192           | type F32  | T+  31\n",
      "[129/723] Writing tensor blk.13.ffn_norm.weight                 | size   8192           | type F32  | T+  31\n",
      "[130/723] Writing tensor blk.14.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  31\n",
      "[131/723] Writing tensor blk.14.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  31\n",
      "[132/723] Writing tensor blk.14.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  31\n",
      "[133/723] Writing tensor blk.14.attn_output.weight              | size   8192 x   8192  | type F16  | T+  31\n",
      "[134/723] Writing tensor blk.14.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  32\n",
      "[135/723] Writing tensor blk.14.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  33\n",
      "[136/723] Writing tensor blk.14.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  33\n",
      "[137/723] Writing tensor blk.14.attn_norm.weight                | size   8192           | type F32  | T+  33\n",
      "[138/723] Writing tensor blk.14.ffn_norm.weight                 | size   8192           | type F32  | T+  33\n",
      "[139/723] Writing tensor blk.15.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  33\n",
      "[140/723] Writing tensor blk.15.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  33\n",
      "[141/723] Writing tensor blk.15.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  33\n",
      "[142/723] Writing tensor blk.15.attn_output.weight              | size   8192 x   8192  | type F16  | T+  33\n",
      "[143/723] Writing tensor blk.15.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  35\n",
      "[144/723] Writing tensor blk.15.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  35\n",
      "[145/723] Writing tensor blk.15.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  35\n",
      "[146/723] Writing tensor blk.15.attn_norm.weight                | size   8192           | type F32  | T+  36\n",
      "[147/723] Writing tensor blk.15.ffn_norm.weight                 | size   8192           | type F32  | T+  36\n",
      "[148/723] Writing tensor blk.16.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  36\n",
      "[149/723] Writing tensor blk.16.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  36\n",
      "[150/723] Writing tensor blk.16.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  36\n",
      "[151/723] Writing tensor blk.16.attn_output.weight              | size   8192 x   8192  | type F16  | T+  36\n",
      "[152/723] Writing tensor blk.16.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  38\n",
      "[153/723] Writing tensor blk.16.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  38\n",
      "[154/723] Writing tensor blk.16.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  38\n",
      "[155/723] Writing tensor blk.16.attn_norm.weight                | size   8192           | type F32  | T+  39\n",
      "[156/723] Writing tensor blk.16.ffn_norm.weight                 | size   8192           | type F32  | T+  39\n",
      "[157/723] Writing tensor blk.17.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  39\n",
      "[158/723] Writing tensor blk.17.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  39\n",
      "[159/723] Writing tensor blk.17.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  39\n",
      "[160/723] Writing tensor blk.17.attn_output.weight              | size   8192 x   8192  | type F16  | T+  39\n",
      "[161/723] Writing tensor blk.17.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  40\n",
      "[162/723] Writing tensor blk.17.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  40\n",
      "[163/723] Writing tensor blk.17.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  41\n",
      "[164/723] Writing tensor blk.17.attn_norm.weight                | size   8192           | type F32  | T+  41\n",
      "[165/723] Writing tensor blk.17.ffn_norm.weight                 | size   8192           | type F32  | T+  41\n",
      "[166/723] Writing tensor blk.18.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  41\n",
      "[167/723] Writing tensor blk.18.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  41\n",
      "[168/723] Writing tensor blk.18.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  41\n",
      "[169/723] Writing tensor blk.18.attn_output.weight              | size   8192 x   8192  | type F16  | T+  41\n",
      "[170/723] Writing tensor blk.18.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  42\n",
      "[171/723] Writing tensor blk.18.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  43\n",
      "[172/723] Writing tensor blk.18.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  43\n",
      "[173/723] Writing tensor blk.18.attn_norm.weight                | size   8192           | type F32  | T+  43\n",
      "[174/723] Writing tensor blk.18.ffn_norm.weight                 | size   8192           | type F32  | T+  43\n",
      "[175/723] Writing tensor blk.19.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  43\n",
      "[176/723] Writing tensor blk.19.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  43\n",
      "[177/723] Writing tensor blk.19.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  43\n",
      "[178/723] Writing tensor blk.19.attn_output.weight              | size   8192 x   8192  | type F16  | T+  43\n",
      "[179/723] Writing tensor blk.19.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  45\n",
      "[180/723] Writing tensor blk.19.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  45\n",
      "[181/723] Writing tensor blk.19.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  45\n",
      "[182/723] Writing tensor blk.19.attn_norm.weight                | size   8192           | type F32  | T+  45\n",
      "[183/723] Writing tensor blk.19.ffn_norm.weight                 | size   8192           | type F32  | T+  45\n",
      "[184/723] Writing tensor blk.20.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  45\n",
      "[185/723] Writing tensor blk.20.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  45\n",
      "[186/723] Writing tensor blk.20.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  45\n",
      "[187/723] Writing tensor blk.20.attn_output.weight              | size   8192 x   8192  | type F16  | T+  45\n",
      "[188/723] Writing tensor blk.20.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  47\n",
      "[189/723] Writing tensor blk.20.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  47\n",
      "[190/723] Writing tensor blk.20.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  47\n",
      "[191/723] Writing tensor blk.20.attn_norm.weight                | size   8192           | type F32  | T+  47\n",
      "[192/723] Writing tensor blk.20.ffn_norm.weight                 | size   8192           | type F32  | T+  47\n",
      "[193/723] Writing tensor blk.21.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  47\n",
      "[194/723] Writing tensor blk.21.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  47\n",
      "[195/723] Writing tensor blk.21.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  47\n",
      "[196/723] Writing tensor blk.21.attn_output.weight              | size   8192 x   8192  | type F16  | T+  47\n",
      "[197/723] Writing tensor blk.21.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  49\n",
      "[198/723] Writing tensor blk.21.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  49\n",
      "[199/723] Writing tensor blk.21.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  50\n",
      "[200/723] Writing tensor blk.21.attn_norm.weight                | size   8192           | type F32  | T+  50\n",
      "[201/723] Writing tensor blk.21.ffn_norm.weight                 | size   8192           | type F32  | T+  50\n",
      "[202/723] Writing tensor blk.22.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  50\n",
      "[203/723] Writing tensor blk.22.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  50\n",
      "[204/723] Writing tensor blk.22.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  50\n",
      "[205/723] Writing tensor blk.22.attn_output.weight              | size   8192 x   8192  | type F16  | T+  50\n",
      "[206/723] Writing tensor blk.22.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  52\n",
      "[207/723] Writing tensor blk.22.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  52\n",
      "[208/723] Writing tensor blk.22.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  52\n",
      "[209/723] Writing tensor blk.22.attn_norm.weight                | size   8192           | type F32  | T+  52\n",
      "[210/723] Writing tensor blk.22.ffn_norm.weight                 | size   8192           | type F32  | T+  52\n",
      "[211/723] Writing tensor blk.23.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  52\n",
      "[212/723] Writing tensor blk.23.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  52\n",
      "[213/723] Writing tensor blk.23.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  52\n",
      "[214/723] Writing tensor blk.23.attn_output.weight              | size   8192 x   8192  | type F16  | T+  52\n",
      "[215/723] Writing tensor blk.23.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  54\n",
      "[216/723] Writing tensor blk.23.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  54\n",
      "[217/723] Writing tensor blk.23.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  55\n",
      "[218/723] Writing tensor blk.23.attn_norm.weight                | size   8192           | type F32  | T+  55\n",
      "[219/723] Writing tensor blk.23.ffn_norm.weight                 | size   8192           | type F32  | T+  55\n",
      "[220/723] Writing tensor blk.24.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  55\n",
      "[221/723] Writing tensor blk.24.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  55\n",
      "[222/723] Writing tensor blk.24.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  55\n",
      "[223/723] Writing tensor blk.24.attn_output.weight              | size   8192 x   8192  | type F16  | T+  55\n",
      "[224/723] Writing tensor blk.24.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  56\n",
      "[225/723] Writing tensor blk.24.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  57\n",
      "[226/723] Writing tensor blk.24.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  57\n",
      "[227/723] Writing tensor blk.24.attn_norm.weight                | size   8192           | type F32  | T+  57\n",
      "[228/723] Writing tensor blk.24.ffn_norm.weight                 | size   8192           | type F32  | T+  57\n",
      "[229/723] Writing tensor blk.25.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  57\n",
      "[230/723] Writing tensor blk.25.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  57\n",
      "[231/723] Writing tensor blk.25.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  57\n",
      "[232/723] Writing tensor blk.25.attn_output.weight              | size   8192 x   8192  | type F16  | T+  57\n",
      "[233/723] Writing tensor blk.25.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  59\n",
      "[234/723] Writing tensor blk.25.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  59\n",
      "[235/723] Writing tensor blk.25.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  59\n",
      "[236/723] Writing tensor blk.25.attn_norm.weight                | size   8192           | type F32  | T+  59\n",
      "[237/723] Writing tensor blk.25.ffn_norm.weight                 | size   8192           | type F32  | T+  59\n",
      "[238/723] Writing tensor blk.26.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  59\n",
      "[239/723] Writing tensor blk.26.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  59\n",
      "[240/723] Writing tensor blk.26.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  59\n",
      "[241/723] Writing tensor blk.26.attn_output.weight              | size   8192 x   8192  | type F16  | T+  59\n",
      "[242/723] Writing tensor blk.26.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  61\n",
      "[243/723] Writing tensor blk.26.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  61\n",
      "[244/723] Writing tensor blk.26.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  61\n",
      "[245/723] Writing tensor blk.26.attn_norm.weight                | size   8192           | type F32  | T+  61\n",
      "[246/723] Writing tensor blk.26.ffn_norm.weight                 | size   8192           | type F32  | T+  61\n",
      "[247/723] Writing tensor blk.27.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  61\n",
      "[248/723] Writing tensor blk.27.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  61\n",
      "[249/723] Writing tensor blk.27.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  61\n",
      "[250/723] Writing tensor blk.27.attn_output.weight              | size   8192 x   8192  | type F16  | T+  61\n",
      "[251/723] Writing tensor blk.27.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  63\n",
      "[252/723] Writing tensor blk.27.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  63\n",
      "[253/723] Writing tensor blk.27.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  63\n",
      "[254/723] Writing tensor blk.27.attn_norm.weight                | size   8192           | type F32  | T+  64\n",
      "[255/723] Writing tensor blk.27.ffn_norm.weight                 | size   8192           | type F32  | T+  64\n",
      "[256/723] Writing tensor blk.28.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  64\n",
      "[257/723] Writing tensor blk.28.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  64\n",
      "[258/723] Writing tensor blk.28.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  64\n",
      "[259/723] Writing tensor blk.28.attn_output.weight              | size   8192 x   8192  | type F16  | T+  64\n",
      "[260/723] Writing tensor blk.28.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  65\n",
      "[261/723] Writing tensor blk.28.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  66\n",
      "[262/723] Writing tensor blk.28.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  66\n",
      "[263/723] Writing tensor blk.28.attn_norm.weight                | size   8192           | type F32  | T+  66\n",
      "[264/723] Writing tensor blk.28.ffn_norm.weight                 | size   8192           | type F32  | T+  66\n",
      "[265/723] Writing tensor blk.29.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  66\n",
      "[266/723] Writing tensor blk.29.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  66\n",
      "[267/723] Writing tensor blk.29.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  66\n",
      "[268/723] Writing tensor blk.29.attn_output.weight              | size   8192 x   8192  | type F16  | T+  66\n",
      "[269/723] Writing tensor blk.29.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  68\n",
      "[270/723] Writing tensor blk.29.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  68\n",
      "[271/723] Writing tensor blk.29.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  68\n",
      "[272/723] Writing tensor blk.29.attn_norm.weight                | size   8192           | type F32  | T+  68\n",
      "[273/723] Writing tensor blk.29.ffn_norm.weight                 | size   8192           | type F32  | T+  68\n",
      "[274/723] Writing tensor blk.30.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  68\n",
      "[275/723] Writing tensor blk.30.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  68\n",
      "[276/723] Writing tensor blk.30.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  68\n",
      "[277/723] Writing tensor blk.30.attn_output.weight              | size   8192 x   8192  | type F16  | T+  68\n",
      "[278/723] Writing tensor blk.30.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  70\n",
      "[279/723] Writing tensor blk.30.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  70\n",
      "[280/723] Writing tensor blk.30.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  70\n",
      "[281/723] Writing tensor blk.30.attn_norm.weight                | size   8192           | type F32  | T+  70\n",
      "[282/723] Writing tensor blk.30.ffn_norm.weight                 | size   8192           | type F32  | T+  70\n",
      "[283/723] Writing tensor blk.31.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  70\n",
      "[284/723] Writing tensor blk.31.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  70\n",
      "[285/723] Writing tensor blk.31.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  70\n",
      "[286/723] Writing tensor blk.31.attn_output.weight              | size   8192 x   8192  | type F16  | T+  70\n",
      "[287/723] Writing tensor blk.31.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  72\n",
      "[288/723] Writing tensor blk.31.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  72\n",
      "[289/723] Writing tensor blk.31.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  72\n",
      "[290/723] Writing tensor blk.31.attn_norm.weight                | size   8192           | type F32  | T+  73\n",
      "[291/723] Writing tensor blk.31.ffn_norm.weight                 | size   8192           | type F32  | T+  73\n",
      "[292/723] Writing tensor blk.32.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  73\n",
      "[293/723] Writing tensor blk.32.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  73\n",
      "[294/723] Writing tensor blk.32.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  73\n",
      "[295/723] Writing tensor blk.32.attn_output.weight              | size   8192 x   8192  | type F16  | T+  73\n",
      "[296/723] Writing tensor blk.32.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  74\n",
      "[297/723] Writing tensor blk.32.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  74\n",
      "[298/723] Writing tensor blk.32.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  75\n",
      "[299/723] Writing tensor blk.32.attn_norm.weight                | size   8192           | type F32  | T+  75\n",
      "[300/723] Writing tensor blk.32.ffn_norm.weight                 | size   8192           | type F32  | T+  75\n",
      "[301/723] Writing tensor blk.33.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  75\n",
      "[302/723] Writing tensor blk.33.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  75\n",
      "[303/723] Writing tensor blk.33.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  75\n",
      "[304/723] Writing tensor blk.33.attn_output.weight              | size   8192 x   8192  | type F16  | T+  75\n",
      "[305/723] Writing tensor blk.33.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  77\n",
      "[306/723] Writing tensor blk.33.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  77\n",
      "[307/723] Writing tensor blk.33.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  77\n",
      "[308/723] Writing tensor blk.33.attn_norm.weight                | size   8192           | type F32  | T+  77\n",
      "[309/723] Writing tensor blk.33.ffn_norm.weight                 | size   8192           | type F32  | T+  77\n",
      "[310/723] Writing tensor blk.34.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  77\n",
      "[311/723] Writing tensor blk.34.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  77\n",
      "[312/723] Writing tensor blk.34.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  77\n",
      "[313/723] Writing tensor blk.34.attn_output.weight              | size   8192 x   8192  | type F16  | T+  77\n",
      "[314/723] Writing tensor blk.34.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  79\n",
      "[315/723] Writing tensor blk.34.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  79\n",
      "[316/723] Writing tensor blk.34.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  79\n",
      "[317/723] Writing tensor blk.34.attn_norm.weight                | size   8192           | type F32  | T+  79\n",
      "[318/723] Writing tensor blk.34.ffn_norm.weight                 | size   8192           | type F32  | T+  79\n",
      "[319/723] Writing tensor blk.35.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  79\n",
      "[320/723] Writing tensor blk.35.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  80\n",
      "[321/723] Writing tensor blk.35.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  80\n",
      "[322/723] Writing tensor blk.35.attn_output.weight              | size   8192 x   8192  | type F16  | T+  80\n",
      "[323/723] Writing tensor blk.35.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  81\n",
      "[324/723] Writing tensor blk.35.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  81\n",
      "[325/723] Writing tensor blk.35.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  82\n",
      "[326/723] Writing tensor blk.35.attn_norm.weight                | size   8192           | type F32  | T+  82\n",
      "[327/723] Writing tensor blk.35.ffn_norm.weight                 | size   8192           | type F32  | T+  82\n",
      "[328/723] Writing tensor blk.36.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  82\n",
      "[329/723] Writing tensor blk.36.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  82\n",
      "[330/723] Writing tensor blk.36.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  82\n",
      "[331/723] Writing tensor blk.36.attn_output.weight              | size   8192 x   8192  | type F16  | T+  82\n",
      "[332/723] Writing tensor blk.36.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  83\n",
      "[333/723] Writing tensor blk.36.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  84\n",
      "[334/723] Writing tensor blk.36.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  84\n",
      "[335/723] Writing tensor blk.36.attn_norm.weight                | size   8192           | type F32  | T+  84\n",
      "[336/723] Writing tensor blk.36.ffn_norm.weight                 | size   8192           | type F32  | T+  84\n",
      "[337/723] Writing tensor blk.37.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  84\n",
      "[338/723] Writing tensor blk.37.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  85\n",
      "[339/723] Writing tensor blk.37.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  85\n",
      "[340/723] Writing tensor blk.37.attn_output.weight              | size   8192 x   8192  | type F16  | T+  85\n",
      "[341/723] Writing tensor blk.37.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  86\n",
      "[342/723] Writing tensor blk.37.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  86\n",
      "[343/723] Writing tensor blk.37.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  86\n",
      "[344/723] Writing tensor blk.37.attn_norm.weight                | size   8192           | type F32  | T+  87\n",
      "[345/723] Writing tensor blk.37.ffn_norm.weight                 | size   8192           | type F32  | T+  87\n",
      "[346/723] Writing tensor blk.38.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  87\n",
      "[347/723] Writing tensor blk.38.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  87\n",
      "[348/723] Writing tensor blk.38.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  87\n",
      "[349/723] Writing tensor blk.38.attn_output.weight              | size   8192 x   8192  | type F16  | T+  87\n",
      "[350/723] Writing tensor blk.38.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  88\n",
      "[351/723] Writing tensor blk.38.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  88\n",
      "[352/723] Writing tensor blk.38.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  89\n",
      "[353/723] Writing tensor blk.38.attn_norm.weight                | size   8192           | type F32  | T+  89\n",
      "[354/723] Writing tensor blk.38.ffn_norm.weight                 | size   8192           | type F32  | T+  89\n",
      "[355/723] Writing tensor blk.39.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  89\n",
      "[356/723] Writing tensor blk.39.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  89\n",
      "[357/723] Writing tensor blk.39.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  89\n",
      "[358/723] Writing tensor blk.39.attn_output.weight              | size   8192 x   8192  | type F16  | T+  89\n",
      "[359/723] Writing tensor blk.39.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  90\n",
      "[360/723] Writing tensor blk.39.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  91\n",
      "[361/723] Writing tensor blk.39.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  91\n",
      "[362/723] Writing tensor blk.39.attn_norm.weight                | size   8192           | type F32  | T+  91\n",
      "[363/723] Writing tensor blk.39.ffn_norm.weight                 | size   8192           | type F32  | T+  91\n",
      "[364/723] Writing tensor blk.40.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  91\n",
      "[365/723] Writing tensor blk.40.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  91\n",
      "[366/723] Writing tensor blk.40.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  91\n",
      "[367/723] Writing tensor blk.40.attn_output.weight              | size   8192 x   8192  | type F16  | T+  91\n",
      "[368/723] Writing tensor blk.40.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  93\n",
      "[369/723] Writing tensor blk.40.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  93\n",
      "[370/723] Writing tensor blk.40.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  93\n",
      "[371/723] Writing tensor blk.40.attn_norm.weight                | size   8192           | type F32  | T+  93\n",
      "[372/723] Writing tensor blk.40.ffn_norm.weight                 | size   8192           | type F32  | T+  93\n",
      "[373/723] Writing tensor blk.41.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  93\n",
      "[374/723] Writing tensor blk.41.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  93\n",
      "[375/723] Writing tensor blk.41.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  93\n",
      "[376/723] Writing tensor blk.41.attn_output.weight              | size   8192 x   8192  | type F16  | T+  93\n",
      "[377/723] Writing tensor blk.41.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  95\n",
      "[378/723] Writing tensor blk.41.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  95\n",
      "[379/723] Writing tensor blk.41.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  95\n",
      "[380/723] Writing tensor blk.41.attn_norm.weight                | size   8192           | type F32  | T+  95\n",
      "[381/723] Writing tensor blk.41.ffn_norm.weight                 | size   8192           | type F32  | T+  95\n",
      "[382/723] Writing tensor blk.42.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  95\n",
      "[383/723] Writing tensor blk.42.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  95\n",
      "[384/723] Writing tensor blk.42.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  95\n",
      "[385/723] Writing tensor blk.42.attn_output.weight              | size   8192 x   8192  | type F16  | T+  95\n",
      "[386/723] Writing tensor blk.42.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  97\n",
      "[387/723] Writing tensor blk.42.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+  97\n",
      "[388/723] Writing tensor blk.42.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+  97\n",
      "[389/723] Writing tensor blk.42.attn_norm.weight                | size   8192           | type F32  | T+  98\n",
      "[390/723] Writing tensor blk.42.ffn_norm.weight                 | size   8192           | type F32  | T+  98\n",
      "[391/723] Writing tensor blk.43.attn_q.weight                   | size   8192 x   8192  | type F16  | T+  98\n",
      "[392/723] Writing tensor blk.43.attn_k.weight                   | size   1024 x   8192  | type F16  | T+  98\n",
      "[393/723] Writing tensor blk.43.attn_v.weight                   | size   1024 x   8192  | type F16  | T+  98\n",
      "[394/723] Writing tensor blk.43.attn_output.weight              | size   8192 x   8192  | type F16  | T+  98\n",
      "[395/723] Writing tensor blk.43.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+  99\n",
      "[396/723] Writing tensor blk.43.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 100\n",
      "[397/723] Writing tensor blk.43.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 100\n",
      "[398/723] Writing tensor blk.43.attn_norm.weight                | size   8192           | type F32  | T+ 100\n",
      "[399/723] Writing tensor blk.43.ffn_norm.weight                 | size   8192           | type F32  | T+ 100\n",
      "[400/723] Writing tensor blk.44.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 100\n",
      "[401/723] Writing tensor blk.44.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 100\n",
      "[402/723] Writing tensor blk.44.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 100\n",
      "[403/723] Writing tensor blk.44.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 100\n",
      "[404/723] Writing tensor blk.44.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 102\n",
      "[405/723] Writing tensor blk.44.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 102\n",
      "[406/723] Writing tensor blk.44.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 102\n",
      "[407/723] Writing tensor blk.44.attn_norm.weight                | size   8192           | type F32  | T+ 102\n",
      "[408/723] Writing tensor blk.44.ffn_norm.weight                 | size   8192           | type F32  | T+ 102\n",
      "[409/723] Writing tensor blk.45.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 102\n",
      "[410/723] Writing tensor blk.45.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 102\n",
      "[411/723] Writing tensor blk.45.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 102\n",
      "[412/723] Writing tensor blk.45.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 102\n",
      "[413/723] Writing tensor blk.45.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 104\n",
      "[414/723] Writing tensor blk.45.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 104\n",
      "[415/723] Writing tensor blk.45.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 104\n",
      "[416/723] Writing tensor blk.45.attn_norm.weight                | size   8192           | type F32  | T+ 104\n",
      "[417/723] Writing tensor blk.45.ffn_norm.weight                 | size   8192           | type F32  | T+ 104\n",
      "[418/723] Writing tensor blk.46.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 104\n",
      "[419/723] Writing tensor blk.46.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 105\n",
      "[420/723] Writing tensor blk.46.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 105\n",
      "[421/723] Writing tensor blk.46.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 105\n",
      "[422/723] Writing tensor blk.46.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 106\n",
      "[423/723] Writing tensor blk.46.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 106\n",
      "[424/723] Writing tensor blk.46.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 107\n",
      "[425/723] Writing tensor blk.46.attn_norm.weight                | size   8192           | type F32  | T+ 107\n",
      "[426/723] Writing tensor blk.46.ffn_norm.weight                 | size   8192           | type F32  | T+ 107\n",
      "[427/723] Writing tensor blk.47.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 107\n",
      "[428/723] Writing tensor blk.47.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 107\n",
      "[429/723] Writing tensor blk.47.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 107\n",
      "[430/723] Writing tensor blk.47.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 107\n",
      "[431/723] Writing tensor blk.47.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 108\n",
      "[432/723] Writing tensor blk.47.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 109\n",
      "[433/723] Writing tensor blk.47.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 109\n",
      "[434/723] Writing tensor blk.47.attn_norm.weight                | size   8192           | type F32  | T+ 109\n",
      "[435/723] Writing tensor blk.47.ffn_norm.weight                 | size   8192           | type F32  | T+ 109\n",
      "[436/723] Writing tensor blk.48.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 109\n",
      "[437/723] Writing tensor blk.48.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 109\n",
      "[438/723] Writing tensor blk.48.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 109\n",
      "[439/723] Writing tensor blk.48.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 109\n",
      "[440/723] Writing tensor blk.48.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 111\n",
      "[441/723] Writing tensor blk.48.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 111\n",
      "[442/723] Writing tensor blk.48.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 111\n",
      "[443/723] Writing tensor blk.48.attn_norm.weight                | size   8192           | type F32  | T+ 111\n",
      "[444/723] Writing tensor blk.48.ffn_norm.weight                 | size   8192           | type F32  | T+ 111\n",
      "[445/723] Writing tensor blk.49.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 111\n",
      "[446/723] Writing tensor blk.49.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 111\n",
      "[447/723] Writing tensor blk.49.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 111\n",
      "[448/723] Writing tensor blk.49.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 111\n",
      "[449/723] Writing tensor blk.49.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 113\n",
      "[450/723] Writing tensor blk.49.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 113\n",
      "[451/723] Writing tensor blk.49.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 113\n",
      "[452/723] Writing tensor blk.49.attn_norm.weight                | size   8192           | type F32  | T+ 114\n",
      "[453/723] Writing tensor blk.49.ffn_norm.weight                 | size   8192           | type F32  | T+ 114\n",
      "[454/723] Writing tensor blk.50.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 114\n",
      "[455/723] Writing tensor blk.50.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 114\n",
      "[456/723] Writing tensor blk.50.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 114\n",
      "[457/723] Writing tensor blk.50.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 114\n",
      "[458/723] Writing tensor blk.50.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 115\n",
      "[459/723] Writing tensor blk.50.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 115\n",
      "[460/723] Writing tensor blk.50.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 116\n",
      "[461/723] Writing tensor blk.50.attn_norm.weight                | size   8192           | type F32  | T+ 116\n",
      "[462/723] Writing tensor blk.50.ffn_norm.weight                 | size   8192           | type F32  | T+ 116\n",
      "[463/723] Writing tensor blk.51.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 116\n",
      "[464/723] Writing tensor blk.51.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 116\n",
      "[465/723] Writing tensor blk.51.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 116\n",
      "[466/723] Writing tensor blk.51.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 116\n",
      "[467/723] Writing tensor blk.51.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 118\n",
      "[468/723] Writing tensor blk.51.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 118\n",
      "[469/723] Writing tensor blk.51.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 119\n",
      "[470/723] Writing tensor blk.51.attn_norm.weight                | size   8192           | type F32  | T+ 119\n",
      "[471/723] Writing tensor blk.51.ffn_norm.weight                 | size   8192           | type F32  | T+ 119\n",
      "[472/723] Writing tensor blk.52.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 119\n",
      "[473/723] Writing tensor blk.52.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 119\n",
      "[474/723] Writing tensor blk.52.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 119\n",
      "[475/723] Writing tensor blk.52.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 119\n",
      "[476/723] Writing tensor blk.52.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 121\n",
      "[477/723] Writing tensor blk.52.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 121\n",
      "[478/723] Writing tensor blk.52.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 122\n",
      "[479/723] Writing tensor blk.52.attn_norm.weight                | size   8192           | type F32  | T+ 122\n",
      "[480/723] Writing tensor blk.52.ffn_norm.weight                 | size   8192           | type F32  | T+ 122\n",
      "[481/723] Writing tensor blk.53.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 122\n",
      "[482/723] Writing tensor blk.53.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 122\n",
      "[483/723] Writing tensor blk.53.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 122\n",
      "[484/723] Writing tensor blk.53.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 122\n",
      "[485/723] Writing tensor blk.53.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 123\n",
      "[486/723] Writing tensor blk.53.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 124\n",
      "[487/723] Writing tensor blk.53.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 124\n",
      "[488/723] Writing tensor blk.53.attn_norm.weight                | size   8192           | type F32  | T+ 124\n",
      "[489/723] Writing tensor blk.53.ffn_norm.weight                 | size   8192           | type F32  | T+ 124\n",
      "[490/723] Writing tensor blk.54.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 124\n",
      "[491/723] Writing tensor blk.54.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 124\n",
      "[492/723] Writing tensor blk.54.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 124\n",
      "[493/723] Writing tensor blk.54.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 124\n",
      "[494/723] Writing tensor blk.54.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 126\n",
      "[495/723] Writing tensor blk.54.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 126\n",
      "[496/723] Writing tensor blk.54.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 126\n",
      "[497/723] Writing tensor blk.54.attn_norm.weight                | size   8192           | type F32  | T+ 126\n",
      "[498/723] Writing tensor blk.54.ffn_norm.weight                 | size   8192           | type F32  | T+ 126\n",
      "[499/723] Writing tensor blk.55.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 126\n",
      "[500/723] Writing tensor blk.55.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 126\n",
      "[501/723] Writing tensor blk.55.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 126\n",
      "[502/723] Writing tensor blk.55.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 126\n",
      "[503/723] Writing tensor blk.55.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 128\n",
      "[504/723] Writing tensor blk.55.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 128\n",
      "[505/723] Writing tensor blk.55.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 128\n",
      "[506/723] Writing tensor blk.55.attn_norm.weight                | size   8192           | type F32  | T+ 128\n",
      "[507/723] Writing tensor blk.55.ffn_norm.weight                 | size   8192           | type F32  | T+ 128\n",
      "[508/723] Writing tensor blk.56.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 128\n",
      "[509/723] Writing tensor blk.56.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 129\n",
      "[510/723] Writing tensor blk.56.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 129\n",
      "[511/723] Writing tensor blk.56.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 129\n",
      "[512/723] Writing tensor blk.56.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 130\n",
      "[513/723] Writing tensor blk.56.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 131\n",
      "[514/723] Writing tensor blk.56.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 131\n",
      "[515/723] Writing tensor blk.56.attn_norm.weight                | size   8192           | type F32  | T+ 131\n",
      "[516/723] Writing tensor blk.56.ffn_norm.weight                 | size   8192           | type F32  | T+ 131\n",
      "[517/723] Writing tensor blk.57.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 131\n",
      "[518/723] Writing tensor blk.57.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 131\n",
      "[519/723] Writing tensor blk.57.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 131\n",
      "[520/723] Writing tensor blk.57.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 131\n",
      "[521/723] Writing tensor blk.57.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 133\n",
      "[522/723] Writing tensor blk.57.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 133\n",
      "[523/723] Writing tensor blk.57.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 133\n",
      "[524/723] Writing tensor blk.57.attn_norm.weight                | size   8192           | type F32  | T+ 133\n",
      "[525/723] Writing tensor blk.57.ffn_norm.weight                 | size   8192           | type F32  | T+ 133\n",
      "[526/723] Writing tensor blk.58.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 133\n",
      "[527/723] Writing tensor blk.58.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 133\n",
      "[528/723] Writing tensor blk.58.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 133\n",
      "[529/723] Writing tensor blk.58.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 133\n",
      "[530/723] Writing tensor blk.58.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 135\n",
      "[531/723] Writing tensor blk.58.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 135\n",
      "[532/723] Writing tensor blk.58.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 136\n",
      "[533/723] Writing tensor blk.58.attn_norm.weight                | size   8192           | type F32  | T+ 136\n",
      "[534/723] Writing tensor blk.58.ffn_norm.weight                 | size   8192           | type F32  | T+ 136\n",
      "[535/723] Writing tensor blk.59.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 136\n",
      "[536/723] Writing tensor blk.59.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 136\n",
      "[537/723] Writing tensor blk.59.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 136\n",
      "[538/723] Writing tensor blk.59.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 136\n",
      "[539/723] Writing tensor blk.59.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 137\n",
      "[540/723] Writing tensor blk.59.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 138\n",
      "[541/723] Writing tensor blk.59.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 138\n",
      "[542/723] Writing tensor blk.59.attn_norm.weight                | size   8192           | type F32  | T+ 138\n",
      "[543/723] Writing tensor blk.59.ffn_norm.weight                 | size   8192           | type F32  | T+ 138\n",
      "[544/723] Writing tensor blk.60.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 138\n",
      "[545/723] Writing tensor blk.60.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 138\n",
      "[546/723] Writing tensor blk.60.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 138\n",
      "[547/723] Writing tensor blk.60.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 138\n",
      "[548/723] Writing tensor blk.60.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 140\n",
      "[549/723] Writing tensor blk.60.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 140\n",
      "[550/723] Writing tensor blk.60.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 140\n",
      "[551/723] Writing tensor blk.60.attn_norm.weight                | size   8192           | type F32  | T+ 140\n",
      "[552/723] Writing tensor blk.60.ffn_norm.weight                 | size   8192           | type F32  | T+ 140\n",
      "[553/723] Writing tensor blk.61.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 140\n",
      "[554/723] Writing tensor blk.61.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 140\n",
      "[555/723] Writing tensor blk.61.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 140\n",
      "[556/723] Writing tensor blk.61.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 140\n",
      "[557/723] Writing tensor blk.61.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 142\n",
      "[558/723] Writing tensor blk.61.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 142\n",
      "[559/723] Writing tensor blk.61.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 143\n",
      "[560/723] Writing tensor blk.61.attn_norm.weight                | size   8192           | type F32  | T+ 143\n",
      "[561/723] Writing tensor blk.61.ffn_norm.weight                 | size   8192           | type F32  | T+ 143\n",
      "[562/723] Writing tensor blk.62.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 143\n",
      "[563/723] Writing tensor blk.62.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 143\n",
      "[564/723] Writing tensor blk.62.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 143\n",
      "[565/723] Writing tensor blk.62.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 143\n",
      "[566/723] Writing tensor blk.62.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 144\n",
      "[567/723] Writing tensor blk.62.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 145\n",
      "[568/723] Writing tensor blk.62.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 145\n",
      "[569/723] Writing tensor blk.62.attn_norm.weight                | size   8192           | type F32  | T+ 145\n",
      "[570/723] Writing tensor blk.62.ffn_norm.weight                 | size   8192           | type F32  | T+ 145\n",
      "[571/723] Writing tensor blk.63.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 145\n",
      "[572/723] Writing tensor blk.63.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 145\n",
      "[573/723] Writing tensor blk.63.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 145\n",
      "[574/723] Writing tensor blk.63.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 145\n",
      "[575/723] Writing tensor blk.63.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 147\n",
      "[576/723] Writing tensor blk.63.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 147\n",
      "[577/723] Writing tensor blk.63.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 147\n",
      "[578/723] Writing tensor blk.63.attn_norm.weight                | size   8192           | type F32  | T+ 147\n",
      "[579/723] Writing tensor blk.63.ffn_norm.weight                 | size   8192           | type F32  | T+ 147\n",
      "[580/723] Writing tensor blk.64.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 147\n",
      "[581/723] Writing tensor blk.64.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 147\n",
      "[582/723] Writing tensor blk.64.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 147\n",
      "[583/723] Writing tensor blk.64.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 147\n",
      "[584/723] Writing tensor blk.64.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 149\n",
      "[585/723] Writing tensor blk.64.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 149\n",
      "[586/723] Writing tensor blk.64.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 149\n",
      "[587/723] Writing tensor blk.64.attn_norm.weight                | size   8192           | type F32  | T+ 149\n",
      "[588/723] Writing tensor blk.64.ffn_norm.weight                 | size   8192           | type F32  | T+ 149\n",
      "[589/723] Writing tensor blk.65.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 149\n",
      "[590/723] Writing tensor blk.65.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 149\n",
      "[591/723] Writing tensor blk.65.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 149\n",
      "[592/723] Writing tensor blk.65.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 149\n",
      "[593/723] Writing tensor blk.65.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 151\n",
      "[594/723] Writing tensor blk.65.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 151\n",
      "[595/723] Writing tensor blk.65.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 151\n",
      "[596/723] Writing tensor blk.65.attn_norm.weight                | size   8192           | type F32  | T+ 152\n",
      "[597/723] Writing tensor blk.65.ffn_norm.weight                 | size   8192           | type F32  | T+ 152\n",
      "[598/723] Writing tensor blk.66.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 152\n",
      "[599/723] Writing tensor blk.66.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 152\n",
      "[600/723] Writing tensor blk.66.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 152\n",
      "[601/723] Writing tensor blk.66.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 152\n",
      "[602/723] Writing tensor blk.66.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 153\n",
      "[603/723] Writing tensor blk.66.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 154\n",
      "[604/723] Writing tensor blk.66.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 154\n",
      "[605/723] Writing tensor blk.66.attn_norm.weight                | size   8192           | type F32  | T+ 154\n",
      "[606/723] Writing tensor blk.66.ffn_norm.weight                 | size   8192           | type F32  | T+ 154\n",
      "[607/723] Writing tensor blk.67.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 154\n",
      "[608/723] Writing tensor blk.67.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 154\n",
      "[609/723] Writing tensor blk.67.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 154\n",
      "[610/723] Writing tensor blk.67.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 154\n",
      "[611/723] Writing tensor blk.67.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 156\n",
      "[612/723] Writing tensor blk.67.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 156\n",
      "[613/723] Writing tensor blk.67.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 156\n",
      "[614/723] Writing tensor blk.67.attn_norm.weight                | size   8192           | type F32  | T+ 156\n",
      "[615/723] Writing tensor blk.67.ffn_norm.weight                 | size   8192           | type F32  | T+ 156\n",
      "[616/723] Writing tensor blk.68.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 156\n",
      "[617/723] Writing tensor blk.68.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 156\n",
      "[618/723] Writing tensor blk.68.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 156\n",
      "[619/723] Writing tensor blk.68.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 156\n",
      "[620/723] Writing tensor blk.68.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 158\n",
      "[621/723] Writing tensor blk.68.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 158\n",
      "[622/723] Writing tensor blk.68.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 158\n",
      "[623/723] Writing tensor blk.68.attn_norm.weight                | size   8192           | type F32  | T+ 159\n",
      "[624/723] Writing tensor blk.68.ffn_norm.weight                 | size   8192           | type F32  | T+ 159\n",
      "[625/723] Writing tensor blk.69.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 159\n",
      "[626/723] Writing tensor blk.69.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 159\n",
      "[627/723] Writing tensor blk.69.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 159\n",
      "[628/723] Writing tensor blk.69.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 159\n",
      "[629/723] Writing tensor blk.69.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 160\n",
      "[630/723] Writing tensor blk.69.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 161\n",
      "[631/723] Writing tensor blk.69.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 161\n",
      "[632/723] Writing tensor blk.69.attn_norm.weight                | size   8192           | type F32  | T+ 161\n",
      "[633/723] Writing tensor blk.69.ffn_norm.weight                 | size   8192           | type F32  | T+ 161\n",
      "[634/723] Writing tensor blk.70.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 161\n",
      "[635/723] Writing tensor blk.70.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 161\n",
      "[636/723] Writing tensor blk.70.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 161\n",
      "[637/723] Writing tensor blk.70.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 161\n",
      "[638/723] Writing tensor blk.70.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 163\n",
      "[639/723] Writing tensor blk.70.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 163\n",
      "[640/723] Writing tensor blk.70.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 163\n",
      "[641/723] Writing tensor blk.70.attn_norm.weight                | size   8192           | type F32  | T+ 163\n",
      "[642/723] Writing tensor blk.70.ffn_norm.weight                 | size   8192           | type F32  | T+ 163\n",
      "[643/723] Writing tensor blk.71.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 163\n",
      "[644/723] Writing tensor blk.71.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 163\n",
      "[645/723] Writing tensor blk.71.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 163\n",
      "[646/723] Writing tensor blk.71.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 163\n",
      "[647/723] Writing tensor blk.71.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 165\n",
      "[648/723] Writing tensor blk.71.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 165\n",
      "[649/723] Writing tensor blk.71.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 165\n",
      "[650/723] Writing tensor blk.71.attn_norm.weight                | size   8192           | type F32  | T+ 165\n",
      "[651/723] Writing tensor blk.71.ffn_norm.weight                 | size   8192           | type F32  | T+ 165\n",
      "[652/723] Writing tensor blk.72.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 165\n",
      "[653/723] Writing tensor blk.72.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 165\n",
      "[654/723] Writing tensor blk.72.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 165\n",
      "[655/723] Writing tensor blk.72.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 165\n",
      "[656/723] Writing tensor blk.72.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 167\n",
      "[657/723] Writing tensor blk.72.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 167\n",
      "[658/723] Writing tensor blk.72.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 167\n",
      "[659/723] Writing tensor blk.72.attn_norm.weight                | size   8192           | type F32  | T+ 168\n",
      "[660/723] Writing tensor blk.72.ffn_norm.weight                 | size   8192           | type F32  | T+ 168\n",
      "[661/723] Writing tensor blk.73.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 168\n",
      "[662/723] Writing tensor blk.73.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 168\n",
      "[663/723] Writing tensor blk.73.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 168\n",
      "[664/723] Writing tensor blk.73.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 168\n",
      "[665/723] Writing tensor blk.73.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 169\n",
      "[666/723] Writing tensor blk.73.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 169\n",
      "[667/723] Writing tensor blk.73.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 170\n",
      "[668/723] Writing tensor blk.73.attn_norm.weight                | size   8192           | type F32  | T+ 170\n",
      "[669/723] Writing tensor blk.73.ffn_norm.weight                 | size   8192           | type F32  | T+ 170\n",
      "[670/723] Writing tensor blk.74.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 170\n",
      "[671/723] Writing tensor blk.74.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 170\n",
      "[672/723] Writing tensor blk.74.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 170\n",
      "[673/723] Writing tensor blk.74.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 170\n",
      "[674/723] Writing tensor blk.74.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 172\n",
      "[675/723] Writing tensor blk.74.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 172\n",
      "[676/723] Writing tensor blk.74.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 172\n",
      "[677/723] Writing tensor blk.74.attn_norm.weight                | size   8192           | type F32  | T+ 172\n",
      "[678/723] Writing tensor blk.74.ffn_norm.weight                 | size   8192           | type F32  | T+ 172\n",
      "[679/723] Writing tensor blk.75.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 172\n",
      "[680/723] Writing tensor blk.75.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 172\n",
      "[681/723] Writing tensor blk.75.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 172\n",
      "[682/723] Writing tensor blk.75.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 172\n",
      "[683/723] Writing tensor blk.75.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 174\n",
      "[684/723] Writing tensor blk.75.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 174\n",
      "[685/723] Writing tensor blk.75.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 174\n",
      "[686/723] Writing tensor blk.75.attn_norm.weight                | size   8192           | type F32  | T+ 174\n",
      "[687/723] Writing tensor blk.75.ffn_norm.weight                 | size   8192           | type F32  | T+ 174\n",
      "[688/723] Writing tensor blk.76.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 174\n",
      "[689/723] Writing tensor blk.76.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 174\n",
      "[690/723] Writing tensor blk.76.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 174\n",
      "[691/723] Writing tensor blk.76.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 174\n",
      "[692/723] Writing tensor blk.76.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 176\n",
      "[693/723] Writing tensor blk.76.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 176\n",
      "[694/723] Writing tensor blk.76.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 176\n",
      "[695/723] Writing tensor blk.76.attn_norm.weight                | size   8192           | type F32  | T+ 177\n",
      "[696/723] Writing tensor blk.76.ffn_norm.weight                 | size   8192           | type F32  | T+ 177\n",
      "[697/723] Writing tensor blk.77.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 177\n",
      "[698/723] Writing tensor blk.77.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 177\n",
      "[699/723] Writing tensor blk.77.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 177\n",
      "[700/723] Writing tensor blk.77.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 177\n",
      "[701/723] Writing tensor blk.77.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 178\n",
      "[702/723] Writing tensor blk.77.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 179\n",
      "[703/723] Writing tensor blk.77.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 179\n",
      "[704/723] Writing tensor blk.77.attn_norm.weight                | size   8192           | type F32  | T+ 179\n",
      "[705/723] Writing tensor blk.77.ffn_norm.weight                 | size   8192           | type F32  | T+ 179\n",
      "[706/723] Writing tensor blk.78.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 179\n",
      "[707/723] Writing tensor blk.78.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 179\n",
      "[708/723] Writing tensor blk.78.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 179\n",
      "[709/723] Writing tensor blk.78.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 179\n",
      "[710/723] Writing tensor blk.78.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 181\n",
      "[711/723] Writing tensor blk.78.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 181\n",
      "[712/723] Writing tensor blk.78.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 181\n",
      "[713/723] Writing tensor blk.78.attn_norm.weight                | size   8192           | type F32  | T+ 181\n",
      "[714/723] Writing tensor blk.78.ffn_norm.weight                 | size   8192           | type F32  | T+ 181\n",
      "[715/723] Writing tensor blk.79.attn_q.weight                   | size   8192 x   8192  | type F16  | T+ 181\n",
      "[716/723] Writing tensor blk.79.attn_k.weight                   | size   1024 x   8192  | type F16  | T+ 181\n",
      "[717/723] Writing tensor blk.79.attn_v.weight                   | size   1024 x   8192  | type F16  | T+ 181\n",
      "[718/723] Writing tensor blk.79.attn_output.weight              | size   8192 x   8192  | type F16  | T+ 181\n",
      "[719/723] Writing tensor blk.79.ffn_gate.weight                 | size  28672 x   8192  | type F16  | T+ 183\n",
      "[720/723] Writing tensor blk.79.ffn_down.weight                 | size   8192 x  28672  | type F16  | T+ 183\n",
      "[721/723] Writing tensor blk.79.ffn_up.weight                   | size  28672 x   8192  | type F16  | T+ 183\n",
      "[722/723] Writing tensor blk.79.attn_norm.weight                | size   8192           | type F32  | T+ 183\n",
      "[723/723] Writing tensor blk.79.ffn_norm.weight                 | size   8192           | type F32  | T+ 183\n",
      "Wrote models/70B-v2/ggml-model-f16.gguf\n",
      "/bin/bash: line 1: ./quantize: No such file or directory\n",
      "/bin/bash: line 1: ./quantize: No such file or directory\n",
      "/bin/bash: line 1: ./quantize: No such file or directory\n"
     ]
    }
   ],
   "source": [
    "# convert the models to ggml FP16 format\n",
    "!python3 convert.py models/7B-v2/\n",
    "!python3 convert.py models/13B-v2/\n",
    "!python3 convert.py models/70B-v2/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "954d1eb9-d1d6-4525-8b0f-3b5809ad2d84",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I llama.cpp build info: \n",
      "I UNAME_S:   Linux\n",
      "I UNAME_P:   x86_64\n",
      "I UNAME_M:   x86_64\n",
      "I CFLAGS:    -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -pthread -march=native -mtune=native -Wdouble-promotion \n",
      "I CXXFLAGS:  -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi\n",
      "I NVCCFLAGS:  \n",
      "I LDFLAGS:    \n",
      "I CC:        cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\n",
      "I CXX:       g++ (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\n",
      "\n",
      "rm -vrf *.o tests/*.o *.so *.dll benchmark-matmult common/build-info.cpp *.dot *.gcno tests/*.gcno *.gcda tests/*.gcda *.gcov tests/*.gcov lcov-report gcovr-report main quantize quantize-stats perplexity embedding vdot q8dot train-text-from-scratch convert-llama2c-to-ggml simple batched batched-bench save-load-state server gguf llama-bench libllava.a llava-cli baby-llama beam-search speculative infill tokenize benchmark-matmult parallel finetune export-lora lookahead lookup tests/test-c.o metal tests/test-llama-grammar tests/test-grammar-parser tests/test-double-float tests/test-grad0 tests/test-opt tests/test-quantize-fns tests/test-quantize-perf tests/test-sampling tests/test-tokenizer-0-llama tests/test-tokenizer-0-falcon tests/test-tokenizer-1-llama tests/test-tokenizer-1-bpe tests/test-rope tests/test-backend-ops\n",
      "removed 'build-info.o'\n",
      "removed 'common.o'\n",
      "removed 'console.o'\n",
      "removed 'ggml-alloc.o'\n",
      "removed 'ggml-backend.o'\n",
      "removed 'ggml-cuda.o'\n",
      "removed 'ggml-quants.o'\n",
      "removed 'ggml.o'\n",
      "removed 'grammar-parser.o'\n",
      "removed 'llama.o'\n",
      "removed 'sampling.o'\n",
      "removed 'train.o'\n",
      "removed 'tests/test-c.o'\n",
      "removed 'benchmark-matmult'\n",
      "removed 'common/build-info.cpp'\n",
      "removed 'main'\n",
      "removed 'quantize'\n",
      "removed 'perplexity'\n",
      "removed 'embedding'\n",
      "removed 'vdot'\n",
      "removed 'q8dot'\n",
      "removed 'train-text-from-scratch'\n",
      "removed 'convert-llama2c-to-ggml'\n",
      "removed 'simple'\n",
      "removed 'batched'\n",
      "removed 'batched-bench'\n",
      "removed 'save-load-state'\n",
      "removed 'gguf'\n",
      "removed 'libllava.a'\n",
      "removed 'baby-llama'\n",
      "removed 'beam-search'\n",
      "removed 'speculative'\n",
      "removed 'infill'\n",
      "removed 'tokenize'\n",
      "removed 'parallel'\n",
      "removed 'finetune'\n",
      "removed 'export-lora'\n",
      "removed 'lookahead'\n",
      "removed 'lookup'\n",
      "I llama.cpp build info: \n",
      "I UNAME_S:   Linux\n",
      "I UNAME_P:   x86_64\n",
      "I UNAME_M:   x86_64\n",
      "I CFLAGS:    -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -pthread -march=native -mtune=native -Wdouble-promotion \n",
      "I CXXFLAGS:  -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi\n",
      "I NVCCFLAGS: -use_fast_math --forward-unknown-to-host-compiler -arch=native -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_MMV_Y=1 -DK_QUANTS_PER_ITERATION=2 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 \n",
      "I LDFLAGS:   -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "I CC:        cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\n",
      "I CXX:       g++ (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\n",
      "\n",
      "cc  -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -pthread -march=native -mtune=native -Wdouble-promotion    -c ggml.c -o ggml.o\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -c llama.cpp -o llama.o\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -c common/common.cpp -o common.o\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -c common/sampling.cpp -o sampling.o\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -c common/grammar-parser.cpp -o grammar-parser.o\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -c common/console.cpp -o console.o\n",
      "nvcc -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -use_fast_math --forward-unknown-to-host-compiler -arch=native -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_MMV_Y=1 -DK_QUANTS_PER_ITERATION=2 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128  -Wno-pedantic -Xcompiler \"-Wno-array-bounds -Wno-format-truncation -Wextra-semi\" -c ggml-cuda.cu -o ggml-cuda.o\n",
      "cc  -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -pthread -march=native -mtune=native -Wdouble-promotion    -c ggml-alloc.c -o ggml-alloc.o\n",
      "cc  -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -pthread -march=native -mtune=native -Wdouble-promotion    -c ggml-backend.c -o ggml-backend.o\n",
      "cc -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -pthread -march=native -mtune=native -Wdouble-promotion     -c ggml-quants.c -o ggml-quants.o\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -c common/train.cpp -o train.o\n",
      "cc -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c11   -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -pthread -march=native -mtune=native -Wdouble-promotion  -c tests/test-c.c -o tests/test-c.o\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -c common/build-info.cpp -o build-info.o\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi pocs/vdot/vdot.cpp ggml.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o vdot -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi pocs/vdot/q8dot.cpp ggml.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o q8dot -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/gguf/gguf.cpp ggml.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o gguf -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/benchmark/benchmark-matmult.cpp build-info.o ggml.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o benchmark-matmult -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/export-lora/export-lora.cpp ggml.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o export-lora -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/main/main.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o console.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o main -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/quantize/quantize.cpp build-info.o ggml.o llama.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o quantize -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/quantize-stats/quantize-stats.cpp build-info.o ggml.o llama.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o quantize-stats -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/perplexity/perplexity.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o perplexity -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/embedding/embedding.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o embedding -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/train-text-from-scratch/train-text-from-scratch.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o train.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o train-text-from-scratch -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp ggml.o llama.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o convert-llama2c-to-ggml -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/simple/simple.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o simple -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/batched/batched.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o batched -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/batched-bench/batched-bench.cpp build-info.o ggml.o llama.o common.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o batched-bench -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/save-load-state/save-load-state.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o save-load-state -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -Iexamples/server examples/server/server.cpp examples/llava/clip.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o server -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib   -Wno-cast-qual\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/llama-bench/llama-bench.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o llama-bench -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -static -fPIC -c examples/llava/llava.cpp -o libllava.a -Wno-cast-qual\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/llava/llava-cli.cpp examples/llava/clip.cpp examples/llava/llava.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o llava-cli -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib  -Wno-cast-qual\n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/baby-llama/baby-llama.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o train.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o baby-llama -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/beam-search/beam-search.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o beam-search -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/speculative/speculative.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o speculative -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/infill/infill.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o console.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o infill -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/tokenize/tokenize.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o tokenize -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/parallel/parallel.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o parallel -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/finetune/finetune.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o train.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o finetune -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/lookahead/lookahead.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o lookahead -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "g++ -I. -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_CUBLAS -I/usr/local/cuda/include -I/opt/cuda/include -I/targets/x86_64-linux/include -I/usr/local/cuda/targets/aarch64-linux/include  -std=c++11 -fPIC -O3 -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi examples/lookup/lookup.cpp ggml.o llama.o common.o sampling.o grammar-parser.o build-info.o ggml-cuda.o ggml-alloc.o ggml-backend.o ggml-quants.o -o lookup -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L/targets/x86_64-linux/lib -L/usr/local/cuda/targets/aarch64-linux/lib \n",
      "\n",
      "====  Run ./main -h for help.  ====\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# metal build\n",
    "!make clean && LLAMA_CUBLAS=1 make -j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c99bdabe-ce05-4e4a-bb7f-1ad00b66e57e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: quantizing './models/7B-v2/ggml-model-f16.gguf' to './models/7B-v2/ggml-model-q4_0.gguf' as Q4_0\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llama_model_quantize_internal: meta size = 1714336 bytes\n",
      "[   1/ 291]                    token_embd.weight - [ 4096, 32000,     1,     1], type =    f16, quantizing to q4_0 .. size =   250.00 MiB ->    70.31 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[   2/ 291]                   output_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[   3/ 291]                        output.weight - [ 4096, 32000,     1,     1], type =    f16, quantizing to q6_K .. size =   250.00 MiB ->   102.54 MiB | hist: \n",
      "[   4/ 291]                  blk.0.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.034 0.008 0.012 0.019 0.031 0.050 0.084 0.149 0.256 0.150 0.084 0.051 0.031 0.019 0.012 0.010 \n",
      "[   5/ 291]                  blk.0.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.034 0.008 0.013 0.021 0.033 0.054 0.089 0.150 0.226 0.151 0.089 0.054 0.033 0.021 0.013 0.011 \n",
      "[   6/ 291]                  blk.0.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.024 0.036 0.053 0.074 0.096 0.117 0.129 0.117 0.096 0.074 0.053 0.036 0.024 0.020 \n",
      "[   7/ 291]             blk.0.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.035 0.011 0.017 0.028 0.044 0.068 0.100 0.135 0.155 0.135 0.100 0.068 0.044 0.028 0.017 0.014 \n",
      "[   8/ 291]                blk.0.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[   9/ 291]                blk.0.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[  10/ 291]                  blk.0.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  11/ 291]               blk.0.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  12/ 291]                blk.0.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  13/ 291]                  blk.1.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.013 0.022 0.034 0.052 0.074 0.098 0.121 0.132 0.121 0.098 0.074 0.052 0.034 0.022 0.018 \n",
      "[  14/ 291]                  blk.1.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.013 0.022 0.034 0.051 0.074 0.099 0.121 0.132 0.121 0.099 0.074 0.051 0.034 0.022 0.018 \n",
      "[  15/ 291]                  blk.1.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.014 0.023 0.035 0.052 0.073 0.097 0.119 0.130 0.119 0.097 0.074 0.052 0.035 0.023 0.019 \n",
      "[  16/ 291]             blk.1.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.035 0.012 0.020 0.031 0.047 0.070 0.098 0.129 0.146 0.129 0.099 0.070 0.047 0.031 0.020 0.016 \n",
      "[  17/ 291]                blk.1.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  18/ 291]                blk.1.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  19/ 291]                  blk.1.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  20/ 291]               blk.1.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  21/ 291]                blk.1.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  22/ 291]                  blk.2.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.096 0.114 0.122 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[  23/ 291]                  blk.2.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.075 0.097 0.115 0.124 0.115 0.097 0.075 0.055 0.037 0.024 0.020 \n",
      "[  24/ 291]                  blk.2.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.120 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  25/ 291]             blk.2.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  26/ 291]                blk.2.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  27/ 291]                blk.2.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  28/ 291]                  blk.2.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  29/ 291]               blk.2.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  30/ 291]                blk.2.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  31/ 291]                  blk.3.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.113 0.120 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[  32/ 291]                  blk.3.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.113 0.120 0.113 0.096 0.076 0.056 0.038 0.025 0.020 \n",
      "[  33/ 291]                  blk.3.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  34/ 291]             blk.3.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[  35/ 291]                blk.3.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  36/ 291]                blk.3.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  37/ 291]                  blk.3.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  38/ 291]               blk.3.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  39/ 291]                blk.3.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  40/ 291]                  blk.4.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.038 0.025 0.021 \n",
      "[  41/ 291]                  blk.4.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.113 0.120 0.113 0.096 0.076 0.056 0.038 0.025 0.020 \n",
      "[  42/ 291]                  blk.4.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  43/ 291]             blk.4.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  44/ 291]                blk.4.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  45/ 291]                blk.4.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  46/ 291]                  blk.4.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  47/ 291]               blk.4.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  48/ 291]                blk.4.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  49/ 291]                  blk.5.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[  50/ 291]                  blk.5.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.096 0.076 0.056 0.038 0.025 0.020 \n",
      "[  51/ 291]                  blk.5.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  52/ 291]             blk.5.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  53/ 291]                blk.5.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  54/ 291]                blk.5.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  55/ 291]                  blk.5.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  56/ 291]               blk.5.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  57/ 291]                blk.5.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  58/ 291]                  blk.6.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  59/ 291]                  blk.6.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  60/ 291]                  blk.6.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  61/ 291]             blk.6.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  62/ 291]                blk.6.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  63/ 291]                blk.6.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  64/ 291]                  blk.6.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  65/ 291]               blk.6.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  66/ 291]                blk.6.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  67/ 291]                  blk.7.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[  68/ 291]                  blk.7.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  69/ 291]                  blk.7.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  70/ 291]             blk.7.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  71/ 291]                blk.7.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  72/ 291]                blk.7.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  73/ 291]                  blk.7.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  74/ 291]               blk.7.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  75/ 291]                blk.7.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  76/ 291]                  blk.8.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.097 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  77/ 291]                  blk.8.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  78/ 291]                  blk.8.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  79/ 291]             blk.8.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  80/ 291]                blk.8.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  81/ 291]                blk.8.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  82/ 291]                  blk.8.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  83/ 291]               blk.8.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  84/ 291]                blk.8.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  85/ 291]                  blk.9.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  86/ 291]                  blk.9.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  87/ 291]                  blk.9.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  88/ 291]             blk.9.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  89/ 291]                blk.9.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  90/ 291]                blk.9.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  91/ 291]                  blk.9.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  92/ 291]               blk.9.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  93/ 291]                blk.9.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[  94/ 291]                 blk.10.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  95/ 291]                 blk.10.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  96/ 291]                 blk.10.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  97/ 291]            blk.10.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[  98/ 291]               blk.10.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  99/ 291]               blk.10.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 100/ 291]                 blk.10.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 101/ 291]              blk.10.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 102/ 291]               blk.10.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 103/ 291]                 blk.11.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 104/ 291]                 blk.11.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 105/ 291]                 blk.11.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 106/ 291]            blk.11.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 107/ 291]               blk.11.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 108/ 291]               blk.11.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 109/ 291]                 blk.11.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 110/ 291]              blk.11.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 111/ 291]               blk.11.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 112/ 291]                 blk.12.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 113/ 291]                 blk.12.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 114/ 291]                 blk.12.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 115/ 291]            blk.12.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 116/ 291]               blk.12.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 117/ 291]               blk.12.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 118/ 291]                 blk.12.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 119/ 291]              blk.12.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 120/ 291]               blk.12.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 121/ 291]                 blk.13.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 122/ 291]                 blk.13.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 123/ 291]                 blk.13.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 124/ 291]            blk.13.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 125/ 291]               blk.13.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 126/ 291]               blk.13.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 127/ 291]                 blk.13.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 128/ 291]              blk.13.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 129/ 291]               blk.13.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 130/ 291]                 blk.14.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 131/ 291]                 blk.14.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 132/ 291]                 blk.14.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 133/ 291]            blk.14.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 134/ 291]               blk.14.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 135/ 291]               blk.14.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 136/ 291]                 blk.14.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 137/ 291]              blk.14.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 138/ 291]               blk.14.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 139/ 291]                 blk.15.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 140/ 291]                 blk.15.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 141/ 291]                 blk.15.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 142/ 291]            blk.15.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 143/ 291]               blk.15.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 144/ 291]               blk.15.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 145/ 291]                 blk.15.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 146/ 291]              blk.15.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 147/ 291]               blk.15.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 148/ 291]                 blk.16.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 149/ 291]                 blk.16.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 150/ 291]                 blk.16.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 151/ 291]            blk.16.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 152/ 291]               blk.16.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 153/ 291]               blk.16.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 154/ 291]                 blk.16.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 155/ 291]              blk.16.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 156/ 291]               blk.16.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 157/ 291]                 blk.17.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 158/ 291]                 blk.17.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 159/ 291]                 blk.17.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 160/ 291]            blk.17.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 161/ 291]               blk.17.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 162/ 291]               blk.17.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 163/ 291]                 blk.17.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 164/ 291]              blk.17.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 165/ 291]               blk.17.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 166/ 291]                 blk.18.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 167/ 291]                 blk.18.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 168/ 291]                 blk.18.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 169/ 291]            blk.18.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 170/ 291]               blk.18.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 171/ 291]               blk.18.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 172/ 291]                 blk.18.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 173/ 291]              blk.18.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 174/ 291]               blk.18.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 175/ 291]                 blk.19.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 176/ 291]                 blk.19.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 177/ 291]                 blk.19.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.111 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 178/ 291]            blk.19.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 179/ 291]               blk.19.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 180/ 291]               blk.19.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 181/ 291]                 blk.19.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 182/ 291]              blk.19.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 183/ 291]               blk.19.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 184/ 291]                 blk.20.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 185/ 291]                 blk.20.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 186/ 291]                 blk.20.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 187/ 291]            blk.20.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 188/ 291]               blk.20.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 189/ 291]               blk.20.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 190/ 291]                 blk.20.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 191/ 291]              blk.20.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 192/ 291]               blk.20.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 193/ 291]                 blk.21.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 194/ 291]                 blk.21.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 195/ 291]                 blk.21.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 196/ 291]            blk.21.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 197/ 291]               blk.21.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 198/ 291]               blk.21.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 199/ 291]                 blk.21.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 200/ 291]              blk.21.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 201/ 291]               blk.21.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 202/ 291]                 blk.22.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 203/ 291]                 blk.22.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 204/ 291]                 blk.22.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 205/ 291]            blk.22.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 206/ 291]               blk.22.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 207/ 291]               blk.22.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 208/ 291]                 blk.22.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 209/ 291]              blk.22.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 210/ 291]               blk.22.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 211/ 291]                 blk.23.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 212/ 291]                 blk.23.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 213/ 291]                 blk.23.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 214/ 291]            blk.23.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 215/ 291]               blk.23.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 216/ 291]               blk.23.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 217/ 291]                 blk.23.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 218/ 291]              blk.23.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 219/ 291]               blk.23.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 220/ 291]                 blk.24.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 221/ 291]                 blk.24.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 222/ 291]                 blk.24.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 223/ 291]            blk.24.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 224/ 291]               blk.24.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 225/ 291]               blk.24.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 226/ 291]                 blk.24.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 227/ 291]              blk.24.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 228/ 291]               blk.24.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 229/ 291]                 blk.25.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 230/ 291]                 blk.25.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 231/ 291]                 blk.25.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 232/ 291]            blk.25.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 233/ 291]               blk.25.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 234/ 291]               blk.25.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 235/ 291]                 blk.25.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 236/ 291]              blk.25.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 237/ 291]               blk.25.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 238/ 291]                 blk.26.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 239/ 291]                 blk.26.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 240/ 291]                 blk.26.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 241/ 291]            blk.26.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 242/ 291]               blk.26.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 243/ 291]               blk.26.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 244/ 291]                 blk.26.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 245/ 291]              blk.26.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 246/ 291]               blk.26.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 247/ 291]                 blk.27.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 248/ 291]                 blk.27.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 249/ 291]                 blk.27.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 250/ 291]            blk.27.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 251/ 291]               blk.27.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 252/ 291]               blk.27.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 253/ 291]                 blk.27.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 254/ 291]              blk.27.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 255/ 291]               blk.27.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 256/ 291]                 blk.28.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.111 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 257/ 291]                 blk.28.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 258/ 291]                 blk.28.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 259/ 291]            blk.28.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 260/ 291]               blk.28.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 261/ 291]               blk.28.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 262/ 291]                 blk.28.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 263/ 291]              blk.28.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 264/ 291]               blk.28.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 265/ 291]                 blk.29.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 266/ 291]                 blk.29.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 267/ 291]                 blk.29.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 268/ 291]            blk.29.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 269/ 291]               blk.29.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 270/ 291]               blk.29.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 271/ 291]                 blk.29.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 272/ 291]              blk.29.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 273/ 291]               blk.29.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 274/ 291]                 blk.30.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 275/ 291]                 blk.30.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 276/ 291]                 blk.30.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 277/ 291]            blk.30.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 278/ 291]               blk.30.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 279/ 291]               blk.30.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.120 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 280/ 291]                 blk.30.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 281/ 291]              blk.30.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 282/ 291]               blk.30.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 283/ 291]                 blk.31.attn_q.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 284/ 291]                 blk.31.attn_k.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 285/ 291]                 blk.31.attn_v.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 286/ 291]            blk.31.attn_output.weight - [ 4096,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    32.00 MiB ->     9.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 287/ 291]               blk.31.ffn_gate.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 288/ 291]               blk.31.ffn_down.weight - [11008,  4096,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.015 0.023 0.036 0.054 0.075 0.098 0.116 0.124 0.116 0.098 0.075 0.054 0.036 0.023 0.019 \n",
      "[ 289/ 291]                 blk.31.ffn_up.weight - [ 4096, 11008,     1,     1], type =    f16, quantizing to q4_0 .. size =    86.00 MiB ->    24.19 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 290/ 291]              blk.31.attn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "[ 291/ 291]               blk.31.ffn_norm.weight - [ 4096,     1,     1,     1], type =    f32, size =    0.016 MB\n",
      "llama_model_quantize_internal: model size  = 12853.02 MB\n",
      "llama_model_quantize_internal: quant size  =  3647.87 MB\n",
      "llama_model_quantize_internal: hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "\n",
      "main: quantize time = 17181.91 ms\n",
      "main:    total time = 17181.91 ms\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: quantizing './models/13B-v2/ggml-model-f16.gguf' to './models/13B-v2/ggml-model-q4_0.gguf' as Q4_0\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 363 tensors from ./models/13B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 5120\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 40\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 13824\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 40\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 40\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type  f16:  282 tensors\n",
      "llama_model_quantize_internal: meta size = 1718656 bytes\n",
      "[   1/ 363]                    token_embd.weight - [ 5120, 32000,     1,     1], type =    f16, quantizing to q4_0 .. size =   312.50 MiB ->    87.89 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[   2/ 363]                   output_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[   3/ 363]                        output.weight - [ 5120, 32000,     1,     1], type =    f16, quantizing to q6_K .. size =   312.50 MiB ->   128.17 MiB | hist: \n",
      "[   4/ 363]                  blk.0.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.033 0.006 0.009 0.015 0.024 0.041 0.074 0.153 0.317 0.153 0.075 0.041 0.024 0.015 0.009 0.008 \n",
      "[   5/ 363]                  blk.0.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.033 0.006 0.010 0.015 0.025 0.043 0.078 0.158 0.293 0.158 0.078 0.043 0.025 0.015 0.010 0.008 \n",
      "[   6/ 363]                  blk.0.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.014 0.023 0.035 0.053 0.074 0.097 0.118 0.129 0.119 0.098 0.074 0.053 0.035 0.023 0.019 \n",
      "[   7/ 363]             blk.0.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.035 0.012 0.020 0.031 0.048 0.071 0.099 0.127 0.142 0.127 0.099 0.071 0.048 0.031 0.020 0.016 \n",
      "[   8/ 363]                blk.0.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[   9/ 363]                blk.0.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[  10/ 363]                  blk.0.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.076 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  11/ 363]               blk.0.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  12/ 363]                blk.0.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  13/ 363]                  blk.1.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.013 0.021 0.033 0.050 0.072 0.098 0.124 0.139 0.124 0.098 0.072 0.050 0.033 0.021 0.017 \n",
      "[  14/ 363]                  blk.1.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.013 0.020 0.032 0.049 0.072 0.099 0.125 0.139 0.126 0.099 0.072 0.049 0.032 0.020 0.017 \n",
      "[  15/ 363]                  blk.1.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.024 0.037 0.054 0.075 0.097 0.116 0.124 0.116 0.097 0.075 0.054 0.037 0.024 0.020 \n",
      "[  16/ 363]             blk.1.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.013 0.021 0.033 0.051 0.073 0.099 0.123 0.134 0.123 0.099 0.073 0.051 0.034 0.021 0.018 \n",
      "[  17/ 363]                blk.1.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  18/ 363]                blk.1.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  19/ 363]                  blk.1.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  20/ 363]               blk.1.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  21/ 363]                blk.1.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  22/ 363]                  blk.2.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.076 0.097 0.114 0.123 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[  23/ 363]                  blk.2.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.075 0.097 0.115 0.124 0.115 0.097 0.075 0.055 0.037 0.024 0.020 \n",
      "[  24/ 363]                  blk.2.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  25/ 363]             blk.2.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.024 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.024 0.020 \n",
      "[  26/ 363]                blk.2.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  27/ 363]                blk.2.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  28/ 363]                  blk.2.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  29/ 363]               blk.2.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  30/ 363]                blk.2.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  31/ 363]                  blk.3.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.096 0.113 0.121 0.113 0.097 0.076 0.055 0.038 0.025 0.020 \n",
      "[  32/ 363]                  blk.3.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.075 0.097 0.115 0.123 0.115 0.097 0.076 0.055 0.037 0.024 0.020 \n",
      "[  33/ 363]                  blk.3.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  34/ 363]             blk.3.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[  35/ 363]                blk.3.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  36/ 363]                blk.3.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  37/ 363]                  blk.3.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  38/ 363]               blk.3.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  39/ 363]                blk.3.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  40/ 363]                  blk.4.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[  41/ 363]                  blk.4.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.113 0.120 0.113 0.096 0.076 0.056 0.038 0.025 0.020 \n",
      "[  42/ 363]                  blk.4.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.038 0.025 0.021 \n",
      "[  43/ 363]             blk.4.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  44/ 363]                blk.4.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  45/ 363]                blk.4.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  46/ 363]                  blk.4.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  47/ 363]               blk.4.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  48/ 363]                blk.4.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  49/ 363]                  blk.5.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  50/ 363]                  blk.5.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[  51/ 363]                  blk.5.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  52/ 363]             blk.5.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  53/ 363]                blk.5.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  54/ 363]                blk.5.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  55/ 363]                  blk.5.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  56/ 363]               blk.5.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  57/ 363]                blk.5.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  58/ 363]                  blk.6.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  59/ 363]                  blk.6.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[  60/ 363]                  blk.6.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  61/ 363]             blk.6.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  62/ 363]                blk.6.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  63/ 363]                blk.6.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  64/ 363]                  blk.6.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  65/ 363]               blk.6.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  66/ 363]                blk.6.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  67/ 363]                  blk.7.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[  68/ 363]                  blk.7.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[  69/ 363]                  blk.7.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  70/ 363]             blk.7.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  71/ 363]                blk.7.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  72/ 363]                blk.7.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  73/ 363]                  blk.7.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  74/ 363]               blk.7.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  75/ 363]                blk.7.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  76/ 363]                  blk.8.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  77/ 363]                  blk.8.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  78/ 363]                  blk.8.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  79/ 363]             blk.8.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  80/ 363]                blk.8.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  81/ 363]                blk.8.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  82/ 363]                  blk.8.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  83/ 363]               blk.8.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  84/ 363]                blk.8.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  85/ 363]                  blk.9.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  86/ 363]                  blk.9.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  87/ 363]                  blk.9.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  88/ 363]             blk.9.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  89/ 363]                blk.9.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  90/ 363]                blk.9.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  91/ 363]                  blk.9.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  92/ 363]               blk.9.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  93/ 363]                blk.9.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[  94/ 363]                 blk.10.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  95/ 363]                 blk.10.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  96/ 363]                 blk.10.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[  97/ 363]            blk.10.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  98/ 363]               blk.10.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  99/ 363]               blk.10.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 100/ 363]                 blk.10.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 101/ 363]              blk.10.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 102/ 363]               blk.10.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 103/ 363]                 blk.11.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 104/ 363]                 blk.11.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 105/ 363]                 blk.11.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 106/ 363]            blk.11.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 107/ 363]               blk.11.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 108/ 363]               blk.11.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 109/ 363]                 blk.11.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 110/ 363]              blk.11.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 111/ 363]               blk.11.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 112/ 363]                 blk.12.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 113/ 363]                 blk.12.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 114/ 363]                 blk.12.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 115/ 363]            blk.12.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 116/ 363]               blk.12.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 117/ 363]               blk.12.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 118/ 363]                 blk.12.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 119/ 363]              blk.12.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 120/ 363]               blk.12.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 121/ 363]                 blk.13.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 122/ 363]                 blk.13.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 123/ 363]                 blk.13.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 124/ 363]            blk.13.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 125/ 363]               blk.13.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 126/ 363]               blk.13.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 127/ 363]                 blk.13.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 128/ 363]              blk.13.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 129/ 363]               blk.13.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 130/ 363]                 blk.14.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 131/ 363]                 blk.14.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 132/ 363]                 blk.14.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 133/ 363]            blk.14.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 134/ 363]               blk.14.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 135/ 363]               blk.14.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 136/ 363]                 blk.14.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 137/ 363]              blk.14.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 138/ 363]               blk.14.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 139/ 363]                 blk.15.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 140/ 363]                 blk.15.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 141/ 363]                 blk.15.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 142/ 363]            blk.15.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 143/ 363]               blk.15.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 144/ 363]               blk.15.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 145/ 363]                 blk.15.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 146/ 363]              blk.15.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 147/ 363]               blk.15.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 148/ 363]                 blk.16.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 149/ 363]                 blk.16.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 150/ 363]                 blk.16.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 151/ 363]            blk.16.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 152/ 363]               blk.16.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 153/ 363]               blk.16.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 154/ 363]                 blk.16.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 155/ 363]              blk.16.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 156/ 363]               blk.16.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 157/ 363]                 blk.17.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 158/ 363]                 blk.17.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 159/ 363]                 blk.17.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 160/ 363]            blk.17.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 161/ 363]               blk.17.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 162/ 363]               blk.17.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 163/ 363]                 blk.17.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 164/ 363]              blk.17.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 165/ 363]               blk.17.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 166/ 363]                 blk.18.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 167/ 363]                 blk.18.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 168/ 363]                 blk.18.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 169/ 363]            blk.18.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 170/ 363]               blk.18.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 171/ 363]               blk.18.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 172/ 363]                 blk.18.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 173/ 363]              blk.18.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 174/ 363]               blk.18.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 175/ 363]                 blk.19.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 176/ 363]                 blk.19.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 177/ 363]                 blk.19.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 178/ 363]            blk.19.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 179/ 363]               blk.19.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 180/ 363]               blk.19.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 181/ 363]                 blk.19.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 182/ 363]              blk.19.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 183/ 363]               blk.19.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 184/ 363]                 blk.20.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 185/ 363]                 blk.20.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 186/ 363]                 blk.20.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 187/ 363]            blk.20.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 188/ 363]               blk.20.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 189/ 363]               blk.20.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 190/ 363]                 blk.20.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 191/ 363]              blk.20.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 192/ 363]               blk.20.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 193/ 363]                 blk.21.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 194/ 363]                 blk.21.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 195/ 363]                 blk.21.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 196/ 363]            blk.21.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 197/ 363]               blk.21.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 198/ 363]               blk.21.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 199/ 363]                 blk.21.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 200/ 363]              blk.21.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 201/ 363]               blk.21.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 202/ 363]                 blk.22.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 203/ 363]                 blk.22.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 204/ 363]                 blk.22.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 205/ 363]            blk.22.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 206/ 363]               blk.22.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 207/ 363]               blk.22.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 208/ 363]                 blk.22.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 209/ 363]              blk.22.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 210/ 363]               blk.22.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 211/ 363]                 blk.23.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 212/ 363]                 blk.23.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 213/ 363]                 blk.23.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 214/ 363]            blk.23.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 215/ 363]               blk.23.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 216/ 363]               blk.23.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 217/ 363]                 blk.23.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 218/ 363]              blk.23.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 219/ 363]               blk.23.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 220/ 363]                 blk.24.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 221/ 363]                 blk.24.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 222/ 363]                 blk.24.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 223/ 363]            blk.24.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 224/ 363]               blk.24.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 225/ 363]               blk.24.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 226/ 363]                 blk.24.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 227/ 363]              blk.24.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 228/ 363]               blk.24.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 229/ 363]                 blk.25.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 230/ 363]                 blk.25.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 231/ 363]                 blk.25.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 232/ 363]            blk.25.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 233/ 363]               blk.25.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 234/ 363]               blk.25.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 235/ 363]                 blk.25.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 236/ 363]              blk.25.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 237/ 363]               blk.25.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 238/ 363]                 blk.26.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 239/ 363]                 blk.26.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 240/ 363]                 blk.26.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 241/ 363]            blk.26.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 242/ 363]               blk.26.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 243/ 363]               blk.26.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 244/ 363]                 blk.26.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 245/ 363]              blk.26.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 246/ 363]               blk.26.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 247/ 363]                 blk.27.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 248/ 363]                 blk.27.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 249/ 363]                 blk.27.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 250/ 363]            blk.27.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 251/ 363]               blk.27.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 252/ 363]               blk.27.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 253/ 363]                 blk.27.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 254/ 363]              blk.27.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 255/ 363]               blk.27.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 256/ 363]                 blk.28.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 257/ 363]                 blk.28.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 258/ 363]                 blk.28.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 259/ 363]            blk.28.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 260/ 363]               blk.28.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 261/ 363]               blk.28.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 262/ 363]                 blk.28.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 263/ 363]              blk.28.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 264/ 363]               blk.28.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 265/ 363]                 blk.29.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 266/ 363]                 blk.29.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.112 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 267/ 363]                 blk.29.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 268/ 363]            blk.29.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 269/ 363]               blk.29.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 270/ 363]               blk.29.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 271/ 363]                 blk.29.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 272/ 363]              blk.29.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 273/ 363]               blk.29.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 274/ 363]                 blk.30.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 275/ 363]                 blk.30.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 276/ 363]                 blk.30.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 277/ 363]            blk.30.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 278/ 363]               blk.30.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 279/ 363]               blk.30.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 280/ 363]                 blk.30.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 281/ 363]              blk.30.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 282/ 363]               blk.30.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 283/ 363]                 blk.31.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 284/ 363]                 blk.31.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 285/ 363]                 blk.31.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 286/ 363]            blk.31.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 287/ 363]               blk.31.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 288/ 363]               blk.31.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 289/ 363]                 blk.31.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 290/ 363]              blk.31.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 291/ 363]               blk.31.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 292/ 363]                 blk.32.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 293/ 363]                 blk.32.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 294/ 363]                 blk.32.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 295/ 363]            blk.32.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 296/ 363]               blk.32.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 297/ 363]               blk.32.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 298/ 363]                 blk.32.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 299/ 363]              blk.32.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 300/ 363]               blk.32.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 301/ 363]                 blk.33.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 302/ 363]                 blk.33.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 303/ 363]                 blk.33.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 304/ 363]            blk.33.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 305/ 363]               blk.33.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 306/ 363]               blk.33.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 307/ 363]                 blk.33.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 308/ 363]              blk.33.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 309/ 363]               blk.33.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 310/ 363]                 blk.34.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 311/ 363]                 blk.34.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 312/ 363]                 blk.34.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 313/ 363]            blk.34.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 314/ 363]               blk.34.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 315/ 363]               blk.34.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 316/ 363]                 blk.34.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 317/ 363]              blk.34.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 318/ 363]               blk.34.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 319/ 363]                 blk.35.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 320/ 363]                 blk.35.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 321/ 363]                 blk.35.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 322/ 363]            blk.35.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 323/ 363]               blk.35.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 324/ 363]               blk.35.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 325/ 363]                 blk.35.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 326/ 363]              blk.35.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 327/ 363]               blk.35.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 328/ 363]                 blk.36.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 329/ 363]                 blk.36.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 330/ 363]                 blk.36.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 331/ 363]            blk.36.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 332/ 363]               blk.36.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 333/ 363]               blk.36.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 334/ 363]                 blk.36.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 335/ 363]              blk.36.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 336/ 363]               blk.36.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 337/ 363]                 blk.37.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 338/ 363]                 blk.37.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 339/ 363]                 blk.37.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 340/ 363]            blk.37.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 341/ 363]               blk.37.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 342/ 363]               blk.37.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 343/ 363]                 blk.37.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 344/ 363]              blk.37.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 345/ 363]               blk.37.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 346/ 363]                 blk.38.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 347/ 363]                 blk.38.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 348/ 363]                 blk.38.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 349/ 363]            blk.38.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 350/ 363]               blk.38.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 351/ 363]               blk.38.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 352/ 363]                 blk.38.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 353/ 363]              blk.38.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 354/ 363]               blk.38.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 355/ 363]                 blk.39.attn_q.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 356/ 363]                 blk.39.attn_k.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 357/ 363]                 blk.39.attn_v.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.120 0.112 0.096 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 358/ 363]            blk.39.attn_output.weight - [ 5120,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =    50.00 MiB ->    14.06 MiB | hist: 0.036 0.015 0.025 0.038 0.055 0.076 0.096 0.113 0.121 0.113 0.096 0.076 0.055 0.038 0.025 0.021 \n",
      "[ 359/ 363]               blk.39.ffn_gate.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 360/ 363]               blk.39.ffn_down.weight - [13824,  5120,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.015 0.024 0.037 0.054 0.076 0.098 0.115 0.122 0.115 0.098 0.076 0.054 0.037 0.024 0.020 \n",
      "[ 361/ 363]                 blk.39.ffn_up.weight - [ 5120, 13824,     1,     1], type =    f16, quantizing to q4_0 .. size =   135.00 MiB ->    37.97 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 362/ 363]              blk.39.attn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "[ 363/ 363]               blk.39.ffn_norm.weight - [ 5120,     1,     1,     1], type =    f32, size =    0.020 MB\n",
      "llama_model_quantize_internal: model size  = 24826.58 MB\n",
      "llama_model_quantize_internal: quant size  =  7023.90 MB\n",
      "llama_model_quantize_internal: hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "\n",
      "main: quantize time = 28706.33 ms\n",
      "main:    total time = 28706.33 ms\n",
      "main: build = 1671 (8fe03ff)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: quantizing './models/70B-v2/ggml-model-f16.gguf' to './models/70B-v2/ggml-model-q4_0.gguf' as Q4_0\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 723 tensors from ./models/70B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 8192\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 80\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 28672\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 64\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 8\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - type  f32:  161 tensors\n",
      "llama_model_loader: - type  f16:  562 tensors\n",
      "llama_model_quantize_internal: meta size = 1740160 bytes\n",
      "[   1/ 723]                    token_embd.weight - [ 8192, 32000,     1,     1], type =    f16, quantizing to q4_0 .. size =   500.00 MiB ->   140.62 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.020 \n",
      "[   2/ 723]                   output_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[   3/ 723]                        output.weight - [ 8192, 32000,     1,     1], type =    f16, quantizing to q6_K .. size =   500.00 MiB ->   205.08 MiB | hist: \n",
      "[   4/ 723]                  blk.0.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.034 0.009 0.014 0.023 0.037 0.059 0.093 0.147 0.198 0.148 0.093 0.059 0.037 0.023 0.014 0.012 \n",
      "[   5/ 723]                  blk.0.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.034 0.008 0.013 0.021 0.035 0.057 0.094 0.153 0.201 0.153 0.094 0.057 0.035 0.021 0.013 0.011 \n",
      "[   6/ 723]                  blk.0.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.075 0.096 0.115 0.123 0.115 0.097 0.075 0.055 0.037 0.024 0.020 \n",
      "[   7/ 723]             blk.0.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.014 0.022 0.034 0.052 0.074 0.099 0.120 0.128 0.120 0.099 0.075 0.052 0.035 0.022 0.018 \n",
      "[   8/ 723]                blk.0.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[   9/ 723]                blk.0.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.014 0.023 0.036 0.053 0.075 0.098 0.117 0.125 0.117 0.098 0.075 0.054 0.036 0.023 0.019 \n",
      "[  10/ 723]                  blk.0.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  11/ 723]               blk.0.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  12/ 723]                blk.0.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  13/ 723]                  blk.1.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.035 0.011 0.017 0.028 0.043 0.066 0.099 0.137 0.160 0.137 0.099 0.066 0.043 0.028 0.017 0.015 \n",
      "[  14/ 723]                  blk.1.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.013 0.021 0.033 0.050 0.073 0.099 0.124 0.135 0.124 0.099 0.073 0.050 0.033 0.021 0.018 \n",
      "[  15/ 723]                  blk.1.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.014 0.022 0.033 0.050 0.071 0.097 0.124 0.137 0.124 0.097 0.071 0.050 0.033 0.022 0.018 \n",
      "[  16/ 723]             blk.1.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.014 0.023 0.036 0.054 0.075 0.098 0.116 0.124 0.117 0.098 0.076 0.054 0.036 0.023 0.019 \n",
      "[  17/ 723]                blk.1.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  18/ 723]                blk.1.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  19/ 723]                  blk.1.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  20/ 723]               blk.1.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  21/ 723]                blk.1.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  22/ 723]                  blk.2.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.014 0.022 0.035 0.052 0.075 0.099 0.119 0.127 0.119 0.099 0.075 0.053 0.035 0.022 0.018 \n",
      "[  23/ 723]                  blk.2.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.014 0.023 0.036 0.054 0.075 0.098 0.117 0.125 0.117 0.098 0.075 0.054 0.036 0.023 0.019 \n",
      "[  24/ 723]                  blk.2.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.014 0.023 0.036 0.054 0.075 0.098 0.116 0.124 0.116 0.098 0.075 0.054 0.036 0.023 0.019 \n",
      "[  25/ 723]             blk.2.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  26/ 723]                blk.2.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  27/ 723]                blk.2.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  28/ 723]                  blk.2.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  29/ 723]               blk.2.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  30/ 723]                blk.2.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  31/ 723]                  blk.3.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[  32/ 723]                  blk.3.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.120 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[  33/ 723]                  blk.3.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.055 0.076 0.097 0.113 0.120 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[  34/ 723]             blk.3.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[  35/ 723]                blk.3.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  36/ 723]                blk.3.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  37/ 723]                  blk.3.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  38/ 723]               blk.3.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  39/ 723]                blk.3.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  40/ 723]                  blk.4.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.113 0.097 0.077 0.056 0.038 0.025 0.020 \n",
      "[  41/ 723]                  blk.4.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.024 0.020 \n",
      "[  42/ 723]                  blk.4.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.023 0.036 0.054 0.075 0.098 0.116 0.124 0.116 0.098 0.075 0.054 0.036 0.024 0.020 \n",
      "[  43/ 723]             blk.4.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  44/ 723]                blk.4.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  45/ 723]                blk.4.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  46/ 723]                  blk.4.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  47/ 723]               blk.4.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  48/ 723]                blk.4.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  49/ 723]                  blk.5.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  50/ 723]                  blk.5.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[  51/ 723]                  blk.5.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.121 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[  52/ 723]             blk.5.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[  53/ 723]                blk.5.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  54/ 723]                blk.5.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  55/ 723]                  blk.5.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  56/ 723]               blk.5.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  57/ 723]                blk.5.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  58/ 723]                  blk.6.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  59/ 723]                  blk.6.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[  60/ 723]                  blk.6.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.113 0.120 0.113 0.097 0.076 0.055 0.038 0.025 0.020 \n",
      "[  61/ 723]             blk.6.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[  62/ 723]                blk.6.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  63/ 723]                blk.6.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  64/ 723]                  blk.6.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  65/ 723]               blk.6.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  66/ 723]                blk.6.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  67/ 723]                  blk.7.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  68/ 723]                  blk.7.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  69/ 723]                  blk.7.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[  70/ 723]             blk.7.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[  71/ 723]                blk.7.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  72/ 723]                blk.7.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  73/ 723]                  blk.7.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  74/ 723]               blk.7.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  75/ 723]                blk.7.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  76/ 723]                  blk.8.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  77/ 723]                  blk.8.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  78/ 723]                  blk.8.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[  79/ 723]             blk.8.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  80/ 723]                blk.8.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  81/ 723]                blk.8.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  82/ 723]                  blk.8.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  83/ 723]               blk.8.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  84/ 723]                blk.8.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  85/ 723]                  blk.9.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[  86/ 723]                  blk.9.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[  87/ 723]                  blk.9.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[  88/ 723]             blk.9.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[  89/ 723]                blk.9.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  90/ 723]                blk.9.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[  91/ 723]                  blk.9.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  92/ 723]               blk.9.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  93/ 723]                blk.9.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[  94/ 723]                 blk.10.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[  95/ 723]                 blk.10.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[  96/ 723]                 blk.10.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[  97/ 723]            blk.10.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[  98/ 723]               blk.10.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[  99/ 723]               blk.10.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 100/ 723]                 blk.10.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 101/ 723]              blk.10.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 102/ 723]               blk.10.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 103/ 723]                 blk.11.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 104/ 723]                 blk.11.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 105/ 723]                 blk.11.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 106/ 723]            blk.11.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 107/ 723]               blk.11.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 108/ 723]               blk.11.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 109/ 723]                 blk.11.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 110/ 723]              blk.11.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 111/ 723]               blk.11.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 112/ 723]                 blk.12.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 113/ 723]                 blk.12.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 114/ 723]                 blk.12.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 115/ 723]            blk.12.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 116/ 723]               blk.12.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 117/ 723]               blk.12.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 118/ 723]                 blk.12.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 119/ 723]              blk.12.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 120/ 723]               blk.12.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 121/ 723]                 blk.13.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 122/ 723]                 blk.13.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 123/ 723]                 blk.13.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 124/ 723]            blk.13.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 125/ 723]               blk.13.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 126/ 723]               blk.13.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 127/ 723]                 blk.13.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 128/ 723]              blk.13.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 129/ 723]               blk.13.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 130/ 723]                 blk.14.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 131/ 723]                 blk.14.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 132/ 723]                 blk.14.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 133/ 723]            blk.14.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 134/ 723]               blk.14.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 135/ 723]               blk.14.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 136/ 723]                 blk.14.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 137/ 723]              blk.14.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 138/ 723]               blk.14.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 139/ 723]                 blk.15.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 140/ 723]                 blk.15.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 141/ 723]                 blk.15.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 142/ 723]            blk.15.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 143/ 723]               blk.15.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 144/ 723]               blk.15.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 145/ 723]                 blk.15.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 146/ 723]              blk.15.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 147/ 723]               blk.15.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 148/ 723]                 blk.16.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 149/ 723]                 blk.16.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 150/ 723]                 blk.16.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 151/ 723]            blk.16.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 152/ 723]               blk.16.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 153/ 723]               blk.16.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 154/ 723]                 blk.16.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 155/ 723]              blk.16.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 156/ 723]               blk.16.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 157/ 723]                 blk.17.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 158/ 723]                 blk.17.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.077 0.056 0.038 0.025 0.020 \n",
      "[ 159/ 723]                 blk.17.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 160/ 723]            blk.17.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 161/ 723]               blk.17.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 162/ 723]               blk.17.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 163/ 723]                 blk.17.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 164/ 723]              blk.17.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 165/ 723]               blk.17.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 166/ 723]                 blk.18.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 167/ 723]                 blk.18.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.077 0.056 0.038 0.025 0.020 \n",
      "[ 168/ 723]                 blk.18.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 169/ 723]            blk.18.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 170/ 723]               blk.18.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 171/ 723]               blk.18.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 172/ 723]                 blk.18.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 173/ 723]              blk.18.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 174/ 723]               blk.18.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 175/ 723]                 blk.19.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 176/ 723]                 blk.19.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.096 0.077 0.056 0.038 0.025 0.020 \n",
      "[ 177/ 723]                 blk.19.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.119 0.112 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 178/ 723]            blk.19.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 179/ 723]               blk.19.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 180/ 723]               blk.19.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 181/ 723]                 blk.19.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 182/ 723]              blk.19.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 183/ 723]               blk.19.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 184/ 723]                 blk.20.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 185/ 723]                 blk.20.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 186/ 723]                 blk.20.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.111 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 187/ 723]            blk.20.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 188/ 723]               blk.20.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 189/ 723]               blk.20.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 190/ 723]                 blk.20.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 191/ 723]              blk.20.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 192/ 723]               blk.20.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 193/ 723]                 blk.21.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 194/ 723]                 blk.21.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.120 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 195/ 723]                 blk.21.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 196/ 723]            blk.21.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 197/ 723]               blk.21.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 198/ 723]               blk.21.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 199/ 723]                 blk.21.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 200/ 723]              blk.21.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 201/ 723]               blk.21.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 202/ 723]                 blk.22.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 203/ 723]                 blk.22.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.055 0.076 0.097 0.113 0.120 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 204/ 723]                 blk.22.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 205/ 723]            blk.22.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 206/ 723]               blk.22.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 207/ 723]               blk.22.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 208/ 723]                 blk.22.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 209/ 723]              blk.22.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 210/ 723]               blk.22.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 211/ 723]                 blk.23.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 212/ 723]                 blk.23.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 213/ 723]                 blk.23.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 214/ 723]            blk.23.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 215/ 723]               blk.23.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 216/ 723]               blk.23.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 217/ 723]                 blk.23.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 218/ 723]              blk.23.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 219/ 723]               blk.23.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 220/ 723]                 blk.24.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 221/ 723]                 blk.24.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 222/ 723]                 blk.24.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 223/ 723]            blk.24.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 224/ 723]               blk.24.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 225/ 723]               blk.24.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 226/ 723]                 blk.24.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 227/ 723]              blk.24.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 228/ 723]               blk.24.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 229/ 723]                 blk.25.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 230/ 723]                 blk.25.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 231/ 723]                 blk.25.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 232/ 723]            blk.25.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 233/ 723]               blk.25.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 234/ 723]               blk.25.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 235/ 723]                 blk.25.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 236/ 723]              blk.25.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 237/ 723]               blk.25.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 238/ 723]                 blk.26.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 239/ 723]                 blk.26.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.096 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 240/ 723]                 blk.26.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 241/ 723]            blk.26.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 242/ 723]               blk.26.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 243/ 723]               blk.26.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 244/ 723]                 blk.26.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 245/ 723]              blk.26.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 246/ 723]               blk.26.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 247/ 723]                 blk.27.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 248/ 723]                 blk.27.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.120 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 249/ 723]                 blk.27.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 250/ 723]            blk.27.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 251/ 723]               blk.27.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 252/ 723]               blk.27.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 253/ 723]                 blk.27.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 254/ 723]              blk.27.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 255/ 723]               blk.27.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 256/ 723]                 blk.28.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 257/ 723]                 blk.28.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 258/ 723]                 blk.28.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 259/ 723]            blk.28.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 260/ 723]               blk.28.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 261/ 723]               blk.28.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 262/ 723]                 blk.28.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 263/ 723]              blk.28.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 264/ 723]               blk.28.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 265/ 723]                 blk.29.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 266/ 723]                 blk.29.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 267/ 723]                 blk.29.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 268/ 723]            blk.29.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 269/ 723]               blk.29.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 270/ 723]               blk.29.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 271/ 723]                 blk.29.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 272/ 723]              blk.29.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 273/ 723]               blk.29.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 274/ 723]                 blk.30.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.077 0.056 0.038 0.025 0.020 \n",
      "[ 275/ 723]                 blk.30.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.121 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 276/ 723]                 blk.30.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.112 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 277/ 723]            blk.30.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 278/ 723]               blk.30.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 279/ 723]               blk.30.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 280/ 723]                 blk.30.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 281/ 723]              blk.30.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 282/ 723]               blk.30.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 283/ 723]                 blk.31.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 284/ 723]                 blk.31.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 285/ 723]                 blk.31.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 286/ 723]            blk.31.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 287/ 723]               blk.31.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 288/ 723]               blk.31.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 289/ 723]                 blk.31.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 290/ 723]              blk.31.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 291/ 723]               blk.31.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 292/ 723]                 blk.32.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 293/ 723]                 blk.32.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 294/ 723]                 blk.32.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 295/ 723]            blk.32.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 296/ 723]               blk.32.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 297/ 723]               blk.32.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 298/ 723]                 blk.32.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 299/ 723]              blk.32.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 300/ 723]               blk.32.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 301/ 723]                 blk.33.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 302/ 723]                 blk.33.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 303/ 723]                 blk.33.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 304/ 723]            blk.33.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 305/ 723]               blk.33.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 306/ 723]               blk.33.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 307/ 723]                 blk.33.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 308/ 723]              blk.33.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 309/ 723]               blk.33.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 310/ 723]                 blk.34.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 311/ 723]                 blk.34.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.120 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 312/ 723]                 blk.34.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 313/ 723]            blk.34.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 314/ 723]               blk.34.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 315/ 723]               blk.34.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 316/ 723]                 blk.34.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 317/ 723]              blk.34.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 318/ 723]               blk.34.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 319/ 723]                 blk.35.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 320/ 723]                 blk.35.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 321/ 723]                 blk.35.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 322/ 723]            blk.35.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 323/ 723]               blk.35.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 324/ 723]               blk.35.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 325/ 723]                 blk.35.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 326/ 723]              blk.35.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 327/ 723]               blk.35.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 328/ 723]                 blk.36.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 329/ 723]                 blk.36.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 330/ 723]                 blk.36.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 331/ 723]            blk.36.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 332/ 723]               blk.36.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 333/ 723]               blk.36.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 334/ 723]                 blk.36.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 335/ 723]              blk.36.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 336/ 723]               blk.36.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 337/ 723]                 blk.37.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 338/ 723]                 blk.37.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 339/ 723]                 blk.37.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.076 0.057 0.039 0.025 0.021 \n",
      "[ 340/ 723]            blk.37.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 341/ 723]               blk.37.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 342/ 723]               blk.37.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 343/ 723]                 blk.37.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 344/ 723]              blk.37.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 345/ 723]               blk.37.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 346/ 723]                 blk.38.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 347/ 723]                 blk.38.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 348/ 723]                 blk.38.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 349/ 723]            blk.38.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 350/ 723]               blk.38.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 351/ 723]               blk.38.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 352/ 723]                 blk.38.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 353/ 723]              blk.38.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 354/ 723]               blk.38.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 355/ 723]                 blk.39.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 356/ 723]                 blk.39.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 357/ 723]                 blk.39.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 358/ 723]            blk.39.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 359/ 723]               blk.39.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 360/ 723]               blk.39.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 361/ 723]                 blk.39.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 362/ 723]              blk.39.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 363/ 723]               blk.39.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 364/ 723]                 blk.40.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 365/ 723]                 blk.40.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 366/ 723]                 blk.40.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 367/ 723]            blk.40.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 368/ 723]               blk.40.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 369/ 723]               blk.40.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 370/ 723]                 blk.40.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 371/ 723]              blk.40.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 372/ 723]               blk.40.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 373/ 723]                 blk.41.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 374/ 723]                 blk.41.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 375/ 723]                 blk.41.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 376/ 723]            blk.41.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 377/ 723]               blk.41.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 378/ 723]               blk.41.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 379/ 723]                 blk.41.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 380/ 723]              blk.41.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 381/ 723]               blk.41.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 382/ 723]                 blk.42.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 383/ 723]                 blk.42.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 384/ 723]                 blk.42.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 385/ 723]            blk.42.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 386/ 723]               blk.42.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 387/ 723]               blk.42.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 388/ 723]                 blk.42.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 389/ 723]              blk.42.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 390/ 723]               blk.42.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 391/ 723]                 blk.43.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 392/ 723]                 blk.43.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 393/ 723]                 blk.43.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 394/ 723]            blk.43.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 395/ 723]               blk.43.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 396/ 723]               blk.43.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 397/ 723]                 blk.43.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 398/ 723]              blk.43.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 399/ 723]               blk.43.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 400/ 723]                 blk.44.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 401/ 723]                 blk.44.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 402/ 723]                 blk.44.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 403/ 723]            blk.44.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 404/ 723]               blk.44.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 405/ 723]               blk.44.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 406/ 723]                 blk.44.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 407/ 723]              blk.44.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 408/ 723]               blk.44.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 409/ 723]                 blk.45.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 410/ 723]                 blk.45.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 411/ 723]                 blk.45.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 412/ 723]            blk.45.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 413/ 723]               blk.45.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 414/ 723]               blk.45.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 415/ 723]                 blk.45.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 416/ 723]              blk.45.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 417/ 723]               blk.45.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 418/ 723]                 blk.46.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 419/ 723]                 blk.46.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 420/ 723]                 blk.46.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.111 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 421/ 723]            blk.46.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 422/ 723]               blk.46.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 423/ 723]               blk.46.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 424/ 723]                 blk.46.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 425/ 723]              blk.46.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 426/ 723]               blk.46.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 427/ 723]                 blk.47.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 428/ 723]                 blk.47.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.077 0.056 0.039 0.025 0.020 \n",
      "[ 429/ 723]                 blk.47.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 430/ 723]            blk.47.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 431/ 723]               blk.47.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 432/ 723]               blk.47.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 433/ 723]                 blk.47.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 434/ 723]              blk.47.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 435/ 723]               blk.47.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 436/ 723]                 blk.48.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 437/ 723]                 blk.48.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.121 0.114 0.097 0.076 0.055 0.038 0.025 0.020 \n",
      "[ 438/ 723]                 blk.48.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 439/ 723]            blk.48.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 440/ 723]               blk.48.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 441/ 723]               blk.48.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 442/ 723]                 blk.48.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 443/ 723]              blk.48.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 444/ 723]               blk.48.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 445/ 723]                 blk.49.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 446/ 723]                 blk.49.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.122 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 447/ 723]                 blk.49.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 448/ 723]            blk.49.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 449/ 723]               blk.49.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 450/ 723]               blk.49.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 451/ 723]                 blk.49.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 452/ 723]              blk.49.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 453/ 723]               blk.49.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 454/ 723]                 blk.50.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 455/ 723]                 blk.50.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.123 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 456/ 723]                 blk.50.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 457/ 723]            blk.50.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 458/ 723]               blk.50.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 459/ 723]               blk.50.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 460/ 723]                 blk.50.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 461/ 723]              blk.50.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 462/ 723]               blk.50.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 463/ 723]                 blk.51.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 464/ 723]                 blk.51.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 465/ 723]                 blk.51.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 466/ 723]            blk.51.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 467/ 723]               blk.51.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 468/ 723]               blk.51.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 469/ 723]                 blk.51.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 470/ 723]              blk.51.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 471/ 723]               blk.51.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 472/ 723]                 blk.52.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 473/ 723]                 blk.52.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.077 0.056 0.038 0.025 0.020 \n",
      "[ 474/ 723]                 blk.52.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 475/ 723]            blk.52.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 476/ 723]               blk.52.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 477/ 723]               blk.52.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 478/ 723]                 blk.52.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 479/ 723]              blk.52.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 480/ 723]               blk.52.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 481/ 723]                 blk.53.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 482/ 723]                 blk.53.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.075 0.097 0.115 0.123 0.115 0.097 0.075 0.055 0.037 0.024 0.020 \n",
      "[ 483/ 723]                 blk.53.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 484/ 723]            blk.53.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 485/ 723]               blk.53.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 486/ 723]               blk.53.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 487/ 723]                 blk.53.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 488/ 723]              blk.53.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 489/ 723]               blk.53.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 490/ 723]                 blk.54.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 491/ 723]                 blk.54.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.075 0.097 0.115 0.123 0.115 0.097 0.075 0.055 0.037 0.024 0.020 \n",
      "[ 492/ 723]                 blk.54.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 493/ 723]            blk.54.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 494/ 723]               blk.54.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 495/ 723]               blk.54.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 496/ 723]                 blk.54.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 497/ 723]              blk.54.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 498/ 723]               blk.54.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 499/ 723]                 blk.55.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 500/ 723]                 blk.55.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.121 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 501/ 723]                 blk.55.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 502/ 723]            blk.55.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 503/ 723]               blk.55.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 504/ 723]               blk.55.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 505/ 723]                 blk.55.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 506/ 723]              blk.55.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 507/ 723]               blk.55.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 508/ 723]                 blk.56.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 509/ 723]                 blk.56.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.076 0.097 0.114 0.122 0.114 0.097 0.076 0.055 0.037 0.024 0.020 \n",
      "[ 510/ 723]                 blk.56.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 511/ 723]            blk.56.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 512/ 723]               blk.56.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 513/ 723]               blk.56.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 514/ 723]                 blk.56.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 515/ 723]              blk.56.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 516/ 723]               blk.56.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 517/ 723]                 blk.57.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 518/ 723]                 blk.57.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.075 0.097 0.115 0.123 0.115 0.097 0.075 0.055 0.037 0.024 0.020 \n",
      "[ 519/ 723]                 blk.57.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.076 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 520/ 723]            blk.57.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 521/ 723]               blk.57.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 522/ 723]               blk.57.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 523/ 723]                 blk.57.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 524/ 723]              blk.57.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 525/ 723]               blk.57.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 526/ 723]                 blk.58.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.120 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 527/ 723]                 blk.58.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.023 0.036 0.054 0.075 0.097 0.117 0.126 0.117 0.097 0.075 0.054 0.037 0.023 0.019 \n",
      "[ 528/ 723]                 blk.58.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 529/ 723]            blk.58.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 530/ 723]               blk.58.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 531/ 723]               blk.58.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 532/ 723]                 blk.58.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 533/ 723]              blk.58.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 534/ 723]               blk.58.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 535/ 723]                 blk.59.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 536/ 723]                 blk.59.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.014 0.023 0.037 0.054 0.075 0.097 0.116 0.125 0.116 0.097 0.075 0.054 0.037 0.024 0.019 \n",
      "[ 537/ 723]                 blk.59.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 538/ 723]            blk.59.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 539/ 723]               blk.59.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 540/ 723]               blk.59.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 541/ 723]                 blk.59.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 542/ 723]              blk.59.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 543/ 723]               blk.59.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 544/ 723]                 blk.60.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.024 0.037 0.054 0.075 0.097 0.115 0.123 0.115 0.097 0.076 0.055 0.037 0.024 0.020 \n",
      "[ 545/ 723]                 blk.60.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.013 0.021 0.033 0.050 0.072 0.097 0.123 0.140 0.123 0.097 0.072 0.050 0.034 0.021 0.018 \n",
      "[ 546/ 723]                 blk.60.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 547/ 723]            blk.60.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 548/ 723]               blk.60.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 549/ 723]               blk.60.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 550/ 723]                 blk.60.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 551/ 723]              blk.60.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 552/ 723]               blk.60.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 553/ 723]                 blk.61.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.113 0.120 0.113 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 554/ 723]                 blk.61.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.014 0.023 0.036 0.053 0.074 0.097 0.118 0.129 0.118 0.097 0.074 0.053 0.036 0.023 0.019 \n",
      "[ 555/ 723]                 blk.61.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 556/ 723]            blk.61.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 557/ 723]               blk.61.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 558/ 723]               blk.61.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 559/ 723]                 blk.61.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 560/ 723]              blk.61.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 561/ 723]               blk.61.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 562/ 723]                 blk.62.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.113 0.120 0.113 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 563/ 723]                 blk.62.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.014 0.023 0.036 0.053 0.074 0.097 0.118 0.129 0.118 0.097 0.074 0.053 0.036 0.023 0.019 \n",
      "[ 564/ 723]                 blk.62.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 565/ 723]            blk.62.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 566/ 723]               blk.62.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 567/ 723]               blk.62.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 568/ 723]                 blk.62.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 569/ 723]              blk.62.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 570/ 723]               blk.62.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 571/ 723]                 blk.63.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.121 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 572/ 723]                 blk.63.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.014 0.023 0.035 0.052 0.074 0.097 0.119 0.132 0.119 0.097 0.074 0.053 0.035 0.023 0.019 \n",
      "[ 573/ 723]                 blk.63.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 574/ 723]            blk.63.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 575/ 723]               blk.63.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 576/ 723]               blk.63.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 577/ 723]                 blk.63.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 578/ 723]              blk.63.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 579/ 723]               blk.63.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 580/ 723]                 blk.64.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 581/ 723]                 blk.64.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.036 0.054 0.075 0.097 0.117 0.125 0.117 0.097 0.075 0.054 0.037 0.024 0.019 \n",
      "[ 582/ 723]                 blk.64.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 583/ 723]            blk.64.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 584/ 723]               blk.64.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 585/ 723]               blk.64.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 586/ 723]                 blk.64.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 587/ 723]              blk.64.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 588/ 723]               blk.64.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 589/ 723]                 blk.65.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.076 0.097 0.115 0.122 0.115 0.097 0.076 0.055 0.037 0.024 0.020 \n",
      "[ 590/ 723]                 blk.65.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.013 0.022 0.034 0.051 0.072 0.097 0.122 0.138 0.122 0.097 0.072 0.051 0.034 0.022 0.018 \n",
      "[ 591/ 723]                 blk.65.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 592/ 723]            blk.65.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 593/ 723]               blk.65.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 594/ 723]               blk.65.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 595/ 723]                 blk.65.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 596/ 723]              blk.65.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 597/ 723]               blk.65.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 598/ 723]                 blk.66.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.120 0.113 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 599/ 723]                 blk.66.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.014 0.023 0.036 0.053 0.074 0.097 0.119 0.130 0.119 0.097 0.074 0.053 0.035 0.023 0.019 \n",
      "[ 600/ 723]                 blk.66.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.112 0.117 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 601/ 723]            blk.66.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 602/ 723]               blk.66.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 603/ 723]               blk.66.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 604/ 723]                 blk.66.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 605/ 723]              blk.66.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 606/ 723]               blk.66.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 607/ 723]                 blk.67.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.014 0.023 0.036 0.054 0.075 0.097 0.116 0.125 0.117 0.097 0.075 0.054 0.036 0.023 0.019 \n",
      "[ 608/ 723]                 blk.67.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.014 0.023 0.035 0.053 0.074 0.097 0.119 0.133 0.119 0.096 0.073 0.052 0.035 0.023 0.019 \n",
      "[ 609/ 723]                 blk.67.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 610/ 723]            blk.67.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 611/ 723]               blk.67.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 612/ 723]               blk.67.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 613/ 723]                 blk.67.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 614/ 723]              blk.67.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 615/ 723]               blk.67.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 616/ 723]                 blk.68.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 617/ 723]                 blk.68.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.076 0.097 0.114 0.122 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 618/ 723]                 blk.68.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 619/ 723]            blk.68.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 620/ 723]               blk.68.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 621/ 723]               blk.68.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 622/ 723]                 blk.68.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 623/ 723]              blk.68.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 624/ 723]               blk.68.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 625/ 723]                 blk.69.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 626/ 723]                 blk.69.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.113 0.120 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 627/ 723]                 blk.69.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 628/ 723]            blk.69.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 629/ 723]               blk.69.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 630/ 723]               blk.69.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 631/ 723]                 blk.69.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 632/ 723]              blk.69.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 633/ 723]               blk.69.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 634/ 723]                 blk.70.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 635/ 723]                 blk.70.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.024 0.038 0.055 0.076 0.097 0.114 0.121 0.114 0.097 0.076 0.055 0.038 0.024 0.020 \n",
      "[ 636/ 723]                 blk.70.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.037 0.016 0.025 0.039 0.056 0.076 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 637/ 723]            blk.70.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 638/ 723]               blk.70.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 639/ 723]               blk.70.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 640/ 723]                 blk.70.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 641/ 723]              blk.70.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 642/ 723]               blk.70.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 643/ 723]                 blk.71.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 644/ 723]                 blk.71.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 645/ 723]                 blk.71.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.111 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 646/ 723]            blk.71.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 647/ 723]               blk.71.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 648/ 723]               blk.71.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 649/ 723]                 blk.71.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 650/ 723]              blk.71.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 651/ 723]               blk.71.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 652/ 723]                 blk.72.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 653/ 723]                 blk.72.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 654/ 723]                 blk.72.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 655/ 723]            blk.72.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 656/ 723]               blk.72.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 657/ 723]               blk.72.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 658/ 723]                 blk.72.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.037 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 659/ 723]              blk.72.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 660/ 723]               blk.72.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 661/ 723]                 blk.73.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 662/ 723]                 blk.73.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 663/ 723]                 blk.73.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 664/ 723]            blk.73.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 665/ 723]               blk.73.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 666/ 723]               blk.73.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 667/ 723]                 blk.73.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 668/ 723]              blk.73.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 669/ 723]               blk.73.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 670/ 723]                 blk.74.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 671/ 723]                 blk.74.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 672/ 723]                 blk.74.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 673/ 723]            blk.74.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.115 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 674/ 723]               blk.74.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 675/ 723]               blk.74.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 676/ 723]                 blk.74.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 677/ 723]              blk.74.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 678/ 723]               blk.74.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 679/ 723]                 blk.75.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 680/ 723]                 blk.75.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.113 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 681/ 723]                 blk.75.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.077 0.097 0.112 0.118 0.112 0.096 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 682/ 723]            blk.75.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.116 0.110 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 683/ 723]               blk.75.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 684/ 723]               blk.75.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 685/ 723]                 blk.75.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 686/ 723]              blk.75.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 687/ 723]               blk.75.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 688/ 723]                 blk.76.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.096 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 689/ 723]                 blk.76.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.076 0.096 0.112 0.118 0.112 0.097 0.077 0.056 0.038 0.025 0.021 \n",
      "[ 690/ 723]                 blk.76.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 691/ 723]            blk.76.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 692/ 723]               blk.76.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 693/ 723]               blk.76.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 694/ 723]                 blk.76.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 695/ 723]              blk.76.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 696/ 723]               blk.76.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 697/ 723]                 blk.77.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 698/ 723]                 blk.77.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 699/ 723]                 blk.77.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 700/ 723]            blk.77.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.040 0.057 0.077 0.096 0.110 0.115 0.110 0.096 0.077 0.057 0.040 0.026 0.021 \n",
      "[ 701/ 723]               blk.77.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 702/ 723]               blk.77.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 703/ 723]                 blk.77.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 704/ 723]              blk.77.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 705/ 723]               blk.77.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 706/ 723]                 blk.78.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.039 0.025 0.021 \n",
      "[ 707/ 723]                 blk.78.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.113 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 708/ 723]                 blk.78.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.096 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 709/ 723]            blk.78.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.111 0.116 0.111 0.096 0.077 0.057 0.039 0.026 0.021 \n",
      "[ 710/ 723]               blk.78.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 711/ 723]               blk.78.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 712/ 723]                 blk.78.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.097 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "[ 713/ 723]              blk.78.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 714/ 723]               blk.78.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 715/ 723]                 blk.79.attn_q.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.118 0.112 0.097 0.076 0.056 0.038 0.025 0.021 \n",
      "[ 716/ 723]                 blk.79.attn_k.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.118 0.112 0.096 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 717/ 723]                 blk.79.attn_v.weight - [ 8192,  1024,     1,     1], type =    f16, quantizing to q4_0 .. size =    16.00 MiB ->     4.50 MiB | hist: 0.036 0.015 0.025 0.038 0.056 0.076 0.097 0.112 0.119 0.112 0.097 0.076 0.056 0.038 0.025 0.020 \n",
      "[ 718/ 723]            blk.79.attn_output.weight - [ 8192,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   128.00 MiB ->    36.00 MiB | hist: 0.037 0.016 0.026 0.039 0.057 0.077 0.096 0.110 0.115 0.110 0.096 0.077 0.057 0.040 0.026 0.021 \n",
      "[ 719/ 723]               blk.79.ffn_gate.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.016 0.025 0.039 0.056 0.077 0.097 0.111 0.117 0.111 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 720/ 723]               blk.79.ffn_down.weight - [28672,  8192,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.015 0.024 0.037 0.055 0.076 0.097 0.114 0.122 0.114 0.097 0.076 0.055 0.037 0.024 0.020 \n",
      "[ 721/ 723]                 blk.79.ffn_up.weight - [ 8192, 28672,     1,     1], type =    f16, quantizing to q4_0 .. size =   448.00 MiB ->   126.00 MiB | hist: 0.036 0.015 0.025 0.039 0.056 0.077 0.097 0.112 0.117 0.112 0.097 0.077 0.056 0.039 0.025 0.021 \n",
      "[ 722/ 723]              blk.79.attn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "[ 723/ 723]               blk.79.ffn_norm.weight - [ 8192,     1,     1,     1], type =    f32, size =    0.031 MB\n",
      "llama_model_quantize_internal: model size  = 131565.03 MB\n",
      "llama_model_quantize_internal: quant size  = 37070.73 MB\n",
      "llama_model_quantize_internal: hist: 0.036 0.016 0.025 0.039 0.057 0.077 0.096 0.111 0.117 0.111 0.096 0.077 0.057 0.039 0.025 0.021 \n",
      "\n",
      "main: quantize time = 105771.41 ms\n",
      "main:    total time = 105771.41 ms\n"
     ]
    }
   ],
   "source": [
    "# quantize the model to 4-bits (using q4_0 method)\n",
    "!./quantize ./models/7B-v2/ggml-model-f16.gguf ./models/7B-v2/ggml-model-q4_0.gguf q4_0\n",
    "!./quantize ./models/13B-v2/ggml-model-f16.gguf ./models/13B-v2/ggml-model-q4_0.gguf q4_0\n",
    "!./quantize ./models/70B-v2/ggml-model-f16.gguf ./models/70B-v2/ggml-model-q4_0.gguf q4_0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "416ca561-de1a-4094-ae0b-fd71408d45e6",
   "metadata": {},
   "source": [
    "# inference"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f0aede9-2f19-41e1-bb4f-1a1d30a00156",
   "metadata": {},
   "source": [
    "### 7B Q4_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4c50d2ab-fc82-4119-8ac3-38ead2b8fee8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1691 (7082d24)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703330143\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n",
      "llama_model_loader: loaded meta data with 22 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  21:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q4_0:  225 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = Q4_0\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 3.56 GiB (4.54 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size       =    0.11 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: system memory used  =   70.42 MiB\n",
      "llm_load_tensors: VRAM used           = 3577.55 MiB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 33/33 layers to GPU\n",
      "..................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 256.00 MB\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "llama_new_context_with_model: compute buffer total size = 73.69 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 70.50 MiB\n",
      "llama_new_context_with_model: total VRAM used: 3904.05 MiB (model: 3577.55 MiB, context: 326.50 MiB)\n",
      "\n",
      "system_info: n_threads = 36 / 72 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under this state of things, for twelve years – say rather under thirty-two or thirty-three; for the revolutions of the world run in cycles, of which a very considerable fraction marks the term of human events: under this state of things, then what I call \"the existing order of nature\" had so far decayed that not only can any sort of reform be properly demanded for good government, but the demand itself tends to assume a legally operative character. At least, such was the state of things in the spring of 1834, and of course over our own hemisphere, too. This may seem a roundabout way of coming at the point, which I propose being as direct as may be, but for the moment you must have patience, and put up with these long-winded prefatory crotchets – and if this doesn't strike you as a pleasant method of being bored with reform, it strikes me so. \n",
      "The sun had not yet set; nor was there any shadow in the sky. This was an extraordinary time, and in some ways inexplicable. For instance, no person was ever less like a person, in looks at least, than this man. There was something feeble and dwarfish in his figure; something indescribably vile or odious about his countenance. One could hardly have worked it up to anything human – and yet the effect upon one's sentiment, under all this horror and repulsion, was strangely softened and mollified. One found him (I mean his face) much more interesting than if it had been a good face. It seemed strange that a man so utterly disagreeable to look at should be able in any way whatever to make himself a object of interest or sympathy; but it was nevertheless true. A sort of mysterious, unaccountable something about the man's manner of expression, too, had much to do with this effect: but even without that he would probably have interested me. \n",
      "I said as much to Mrs Clennam. She shook her head. 'It is so easy,' she said, 'to make a foolish person interesting, when he happens to have an interesting face! No doubt he has an interesting voice? I daresay you find him amusing, in that way.'\n",
      "I confessed, reluctantly, that I had not listened long enough yet to say for certain what he said or how he said it; but at the same time I mentioned with a shade of reproof that I did not regard the matter so lightly as she appeared to do. 'You can't judge a book by its cover,' said Mrs Clennam, in an unimpressed manner – 'or anything else for that matter.' She added: 'The only thing we have to ask is whether he really knows what he wants. And if he knows it, is his intention really good? Are there no other reasons why a man should be interesting?'\n",
      "'I can hardly say,' I returned. 'But even supposing all these were in his favour – what then? I have never been able to see any harm in looking at the outside of an apple or an orange.'\n",
      "Her eye met mine again, and she said: 'There is no use talking about it in that way!' She added, with a laugh: 'For a little while I did talk to myself, and got on very nicely – until I was interrupted. I must ask you, however, Mr Vholes – not for any purpose of our own, but as the means of obtaining your opinion in regard to this young lady who is to be Miss Dorothea's companion.'\n",
      "She paused again with an air that showed she expected me to volunteer my opinion. I could not have felt more reluctant about offering it under her present circumstances: I was afraid I should appear a great deal less favourable than I really did feel towards the young lady. Nevertheless, I was obliged to say something; and the only thing I could think of doing (for the moment at least) in order not to be altogether unhelpful, was to mention one or two particular points that struck me as good and others as bad.\n",
      "'There is this young lady,' said Mrs Clennam, 'whose name is Lucilla.'\n",
      "The moment she uttered it, I felt an absurdity of which I had not the least suspicion when she named her. She seemed to have changed places with me in a moment; for, where before I was a stranger speaking about people he saw only at second-hand, and about whose characters (with one exception) he knew no more than is usual under such circumstances, now all of sudden I felt that I could talk about Lucilla. 'This young lady,' said Mrs Clennam, in the same voice as before, 'whose name is Lucilla\n",
      "llama_print_timings:        load time =    1637.42 ms\n",
      "llama_print_timings:      sample time =     581.79 ms /  1024 runs   (    0.57 ms per token,  1760.08 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.57 ms /   494 tokens (    0.38 ms per token,  2605.87 tokens per second)\n",
      "llama_print_timings:        eval time =    8491.38 ms /  1023 runs   (    8.30 ms per token,   120.48 tokens per second)\n",
      "llama_print_timings:       total time =    9573.31 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3e237938-4375-406a-b329-48d6d2363aa9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1691 (7082d24)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703330156\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n",
      "llama_model_loader: loaded meta data with 22 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  21:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q4_0:  225 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = Q4_0\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 3.56 GiB (4.54 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size       =    0.11 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: system memory used  =   70.42 MiB\n",
      "llm_load_tensors: VRAM used           = 3577.55 MiB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 33/33 layers to GPU\n",
      "..................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 256.00 MB\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "llama_new_context_with_model: compute buffer total size = 73.69 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 70.50 MiB\n",
      "llama_new_context_with_model: total VRAM used: 3904.05 MiB (model: 3577.55 MiB, context: 326.50 MiB)\n",
      "\n",
      "system_info: n_threads = 36 / 72 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her star, the French Revolution was in full cry – running its course with a good heart if not much head. The eyes of the world were fixed upon the French army, whose superior officers were careful to mark out the line of march for each subdivision of their troops; anxious above all things that no platoon should take up a position which had not been previously marked out for it. The old nobility stood aghast as the \"have-nots\" began to have and to hold; while the clergy, with their backs to the wall, watch every blow. \n",
      " In England itself, though much less noisy than in France, the storm was brewing – or rather the counterparts of that storm which, having begun by a few private individuals in different parts of the country, at length broke into that mighty hurricane which swept some thousands into their graves, and is still making great havoc among the living. \n",
      " Right Reverend Father in God, Francis of Rome, Bishop of St Peter's, and Dean of Wells; John Palmer, Esq., M. P.; Samuel Egerton Brydges, Esquire; William Smith, LL.D., F. S. A.; Thomas Parkenham, Esq., M. P.; George Walker, Esquire; the Rev. Mr. Watson, and several others of a similar stamp and caliber, are not likely to be forgotten. \n",
      " In this list, which must in some measure stand for all, no special mention occurs of an individual who has rendered services as distinguished as those rendered by any three or four whom I have just named; yet from the public prints it appears that the whole amount of his talents and exertions is not a whit greater than was rendered by each of these, taken singly. \n",
      " It has been remarked in an early part of this volume – what remark? If the reader will turn to page thirty-one, he will find that the passage now under consideration occupies some pages towards the close of the present chapter. The passage in question, if it does not constitute a proof that the author's understanding has become as little exercised by books and study as his senses have been blunted by drinking, may be reckoned among the most unlucky specimens of literary composition that ever issued from press or pen; yet is it one which he will never consent to part with. \n",
      "# CHAPTER III: HERBERT GREATHEART—THE PHYSICIAN IN THE HOUSE OF SINNERS\n",
      "I must do Mr. Greathearth the justice to say that in my interviews with him, at his house and elsewhere, I never found him wanting either in courtesy or liberality; he was a man of excellent address, easy manners, and an agreeable countenance. But as this book is intended merely for a memorial to myself, without being meant to give any other person than Mr. Greathearth himself a character that may possibly be injurious in some degree to his own private interest; and as I have been so fortunate as not only to meet with him at different times during the space of six or eight years past, but also to take part with him in the management of certain charitable funds, which he has been the means of bringing into operation, I should be guilty of a great degree of disingenuity if I did not, at this time, bestow upon him the encomiums that are his due.\n",
      "I had often heard Mr. Greathearth spoken of as one who was richly endowed with natural qualifications and gifts for his calling; but what struck me in a higher degree than any other part of my acquaintance with him was his very uncommon aptitude in the art and mystery of healing, which in these latter times has become almost an artless process. He had also a happy way of concealing his own learning by the simplicity of his demeanour; and, having no vanity whatever about himself or any other man, he seldom indulged in that sort of familiar talk with us young men, which we found very acceptable when it did fall from him, and which we sometimes received with a great deal of seeming diffidence.\n",
      "When I first went to reside at Mr. Greathearth's house, I was only a school-boy; but in a few years I grew up into manhood, and had already begun to practise the art of medicine amongst my poor neighbours, when I found that Mr. G.'s practice in the neighbourhood was almost as large, if not larger than my own, though he resided about five miles distant from me: it appeared that his residence there was so very remote and sequestered, that few people visited him upon business, while they were living so much more near to myself; yet I\n",
      "llama_print_timings:        load time =    1604.99 ms\n",
      "llama_print_timings:      sample time =     565.86 ms /  1024 runs   (    0.55 ms per token,  1809.64 tokens per second)\n",
      "llama_print_timings: prompt eval time =     189.13 ms /   494 tokens (    0.38 ms per token,  2612.02 tokens per second)\n",
      "llama_print_timings:        eval time =    8475.34 ms /  1023 runs   (    8.28 ms per token,   120.70 tokens per second)\n",
      "llama_print_timings:       total time =    9537.79 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "eb2e9a98-ed63-4b0b-b858-8a69c3cde67f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1691 (7082d24)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703330168\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n",
      "llama_model_loader: loaded meta data with 22 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  21:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type q4_0:  225 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = Q4_0\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 3.56 GiB (4.54 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size       =    0.11 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: system memory used  =   70.42 MiB\n",
      "llm_load_tensors: VRAM used           = 3577.55 MiB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 33/33 layers to GPU\n",
      "..................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 256.00 MB\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "llama_new_context_with_model: compute buffer total size = 73.69 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 70.50 MiB\n",
      "llama_new_context_with_model: total VRAM used: 3904.05 MiB (model: 3577.55 MiB, context: 326.50 MiB)\n",
      "\n",
      "system_info: n_threads = 36 / 72 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her good-natured sovereign (the scullion of half-a-dozen kitchens) she mixed business with pleasure, and lovd to have the blues. France was at that epoch what the United States is now — timidly and tentatively, she began to stretch her wings; and down she came, melodiously singing (though with a poor voice), as it were, in the family shake-down, ‘I wish you to understand distinctly that I have no more collected than you could, under similar circumstances.’ \n",
      "All this while Mr. Pickwick was proceeding towards London on his way from Dover to Manchester. Having passed through Rochester and Maidstone he reached the capital in safety; and, after a few days’ delay at the Bell and Horns, set out on his journey. The day for starting was fixed for an early hour next morning; and as Mr. Pickwick always rose by five o’clock, it may be supposed that every precaution to ensure a good start had been taken. Such indeed was the case; but in addition to the customary cautions regarding punctuality, which had been delivered with great severity on all previous occasions, Mr. Pickwick added several of his own composition for the benefit of Mr. Tupman and Mr. Snodgrass, who were also of the party.\n",
      "‘Never mind me,’ said he; ‘I can always stop behind.’\n",
      "‘If I’m not in my place at eight o’clock sharp,’ said Mr. Snodgrass, in a determined tone, ‘I will never be again!’\n",
      "These remarks were made in reference to the dinner-hour, but were intended to impress their companions with the conviction that they should arrive at their destination punctually on all occasions. The effect of this was such as might have been anticipated; for Mr. Tupman and Mr. Snodgrass, upon leaving Mr. Pickwick’s presence, walked very erectly along the street, and assumed an air of great self-importance.\n",
      "‘I hope,’ said Mr. Tupman to his friend, as they walked up Pall Mall together, ‘that this expedition may be successful.’\n",
      "‘And that we shall all return to London in safety!’ responded Mr. Snodgrass, grasping the hand of his companion.\n",
      "‘In safety — but not by way of St. Paul’s,’ replied Mr. Tupman.\n",
      "They were walking along the Strand at this time, and as they came opposite the Criterion bar, a door suddenly flew open in front of them, and out rushed three or four young men, who tumbled through the open doorway into the gutter. It was an awkward accident, for which no one could account, except perhaps that it arose from some gentlemen being rather merry over their wine; but however this might be, Mr. Tupman thought fit to point out to his friend that in such a place they had no business to take a turn so soon after dinner — a remark which his companion very readily acknowledged.\n",
      "‘We have made a most unfortunate start,’ said Mr. Tupman; ‘but, upon the whole, I think we shall do well if we persevere.’\n",
      "Mr. Snodgrass acquiesced in this decision with an air of great self-complacency and importance. The friends parted at Charing Cross, when they went their several ways, and it was not until about a mile from that point that Mr. Tupman began to think himself much mistaken in his calculations as to the probability of their reaching Bath in safety.\n",
      "Their progress had been very slow, for Mr. Tupman kept stopping at every public house he came to, either to take something or give some information; and this brought on many little disputes between him and his companion, who found himself more and more provoked as the day advanced: but this was nothing in comparison of what was still to come, for as they were rambling along a lane near the village of Evesham (Mr. Snodgrass having insisted upon taking it in preference to a road), and within sight of a high hill which he thought looked beautiful through the trees, Mr. Tupman had a sudden impulse to turn off into the park at the foot of the hill; for they were passing by a handsome iron gate, which had no railing to it, so that he fancied that if he went through this unguarded doorway, and then turned suddenly round (as he intended doing), there was some chance of his astonishing Mr. Snodgrass very much — at least more so than by walking in any other direction would have done. So saying to himself ‘Bath or no Bath,\n",
      "llama_print_timings:        load time =    1630.23 ms\n",
      "llama_print_timings:      sample time =     577.48 ms /  1024 runs   (    0.56 ms per token,  1773.22 tokens per second)\n",
      "llama_print_timings: prompt eval time =     190.91 ms /   494 tokens (    0.39 ms per token,  2587.55 tokens per second)\n",
      "llama_print_timings:        eval time =    8481.27 ms /  1023 runs   (    8.29 ms per token,   120.62 tokens per second)\n",
      "llama_print_timings:       total time =    9560.27 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52c40741-4720-463b-a699-a05ae9266ab2",
   "metadata": {},
   "source": [
    "### 7B f16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e3683e44-8c9a-4f85-abfc-37c024375357",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1691 (7082d24)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703330180\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = F16\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 12.55 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size       =    0.11 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: system memory used  =  250.11 MiB\n",
      "llm_load_tensors: VRAM used           = 12603.02 MiB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 33/33 layers to GPU\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 256.00 MB\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "llama_new_context_with_model: compute buffer total size = 73.69 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 70.50 MiB\n",
      "llama_new_context_with_model: total VRAM used: 12929.52 MiB (model: 12603.02 MiB, context: 326.50 MiB)\n",
      "\n",
      "system_info: n_threads = 36 / 72 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under this king and this queen there occurred in the year 1775 the true beginning of that eventful era so memorable in the history of human nature, which has given its name to our own time.\n",
      "I. INTRODUCTORY CHAPTER\n",
      "THE INCIDENTS OF THE SEVERAL DAYS OF MY CHILDHOOD AND YOUTH I have not forgotten them as if they were things removed from me by house and garden and years of living, but often seem in thought to stand within sight of my mother's house or among old haunts and scenes of childish days. For what else is a man who has been born into this world? His mother's arms have held him first; he was borne across his mother's breast long before his mother had ever known that she would bear him, and as he grows older the thoughtful mother sighs sometimes to herself as she thinks how soon her baby will be a man.\n",
      "That is not a true or real life in which there is no misty-magnifying background of dreams, with their shapes of boyhood days, wherein the present picture is set. In our commoner humanity, we have too much sense and wisdom to look forward upon that distant background and try to distinguish those remote figures on its horizon. Yet how many such glimpses we may have in life, and even sometimes see distinctly in a few brief hours of passionate feeling or dream-like revery! How often a man has seemed to himself in the twilight of his life as if he had never lived before. And I have known some people in whose life, as I could observe it, there were just such gaps and lapses from actual facts of time as are sometimes revealed by a sudden shining of the light upon a mountain side and showing how many steps and toiling feet had been required to climb the height; or as are marked on some smooth and level surface where the stone mason has shown the joints by which he laid his stones, to tell how long it took him to do that piece of work.\n",
      "But there is another view from which all this can be taken and which, in truth, is much more common among those who have thoughtfully considered such things as they grow older. It is not only in the lapse of years, but very often in the lapse of months or even days that some people find that life has seemed to them like an actual dream; and it is a far-off misty background of dreams, rather than of facts, which comes into view as their eyes open wider upon this earth. In such cases the light falls first on some brief phase of one’s own character or conduct in years long passed by; it has been remembered and looked back upon as a sort of ideal that we thought ourselves, but really never were; or else, if not this, then perhaps on our relation to others which we once fancied ideal, but which after all was not such. This is the background which the light suddenly shines into upon many passages in one’s own life; and it seems, looking back, as if they had never really lived at all before.\n",
      "There is no doubt that some of these recollections are quite erroneous; but there are others on which the light of years has made me see that I was altogether mistaken in my view of what was ideal and good about them when I was younger and more careless and selfish. As I grew older it seemed as if, somehow or other, all this had changed, though really nothing had been done except the passing of years and a wider insight into life; but it has always seemed to me that there were many things in which I could have bettered myself by remembering those earlier phases of my life with their more romantic idealism.\n",
      "I was looking over a few passages from these journals just now, and I remembered a short note in one of them in which I wrote:\n",
      "“In this age it seems to me that the really important part of life is not work or amusement but the relation of each man to every other; and that it is by studying this relationship—its difficulties, its delights, its tragedy—that we can get a true idea of ourselves and our lives.\n",
      "“It has always seemed to me that there is something better in life than work or amusement, although both are necessary parts of existence; but it seems to be the one thing which we least often look at—the relationship between each individual person in this world with every other, their feelings towards one another and towards themselves. It is these feelings which give us our character and mould our lives, whether good or evil; and they are only known by study of our own hearts as well as others’.”\n",
      "This I wrote down some five-and-twenty years ago—at the end of 1893 to be\n",
      "llama_print_timings:        load time =   12489.78 ms\n",
      "llama_print_timings:      sample time =     578.55 ms /  1024 runs   (    0.56 ms per token,  1769.95 tokens per second)\n",
      "llama_print_timings: prompt eval time =     159.85 ms /   494 tokens (    0.32 ms per token,  3090.47 tokens per second)\n",
      "llama_print_timings:        eval time =   19723.65 ms /  1023 runs   (   19.28 ms per token,    51.87 tokens per second)\n",
      "llama_print_timings:       total time =   20774.16 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "33860261-8154-4b7c-bdec-cccc2626ba86",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1691 (7082d24)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703330215\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = F16\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 12.55 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size       =    0.11 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: system memory used  =  250.11 MiB\n",
      "llm_load_tensors: VRAM used           = 12603.02 MiB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 33/33 layers to GPU\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 256.00 MB\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "llama_new_context_with_model: compute buffer total size = 73.69 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 70.50 MiB\n",
      "llama_new_context_with_model: total VRAM used: 12929.52 MiB (model: 12603.02 MiB, context: 326.50 MiB)\n",
      "\n",
      "system_info: n_threads = 36 / 72 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her Christian pastors, she entertained themselves much with the antic dispensations wherewith they thought it expedient to people their historic earthly stage, turning every thing into a roar of ridicule and sending it out through all the great newspapers of the world. \n",
      " These papers were precise in their inquiry, and minute in their narration. Their children were ushered into different proportions of light, according to the extent of their comprehension! \n",
      " The oldest of them had received lately a rich present of twenty guineas: wherewith he had invited his fellow-prisoners to a lobster salad and a piece of pie: in honour of which he had even compounded some lobsters out of a deal box, and, with that end on't, he had gone down on his marrow bones at the very door of a fish-shop. All the party were present. \n",
      " It was evident that no great mortification awaited the petrel. He had money in his pocket. The piece of pie as high as the moon; and if you want to know more, he had only to whistle for it. His father, indeed (being a respectable man and a farmer), was somewhat backward in sending him this twenty guineas: but still he had got it, and it might be observed that he began with an air of great indifference to do anything for his fellow-creatures; but by little and little his inclination grew. At first he would take no notice of the youngest prisoner; next day he took some notice; third day (which was the lobster salad) he went out, and soon returned with a piece of pie: and this was his own particular manner when anything particularly fine was to be done for his fellow-creature.\n",
      "When it appeared that the gentleman had set apart all his wits and powers upon the great question how he might do something or other for Mr Pickwick (a good sort of man, if ever there was one) at once every body began to consider whether this something should be an entertainment in a coach; whether it should be a feast on board ship; whether it should be a series of dinners and suppers and lunches and teas and coffee-parties and evening assemblies.\n",
      "All the questions, however, were decided by the decision already arrived at touching Mr Pickwick. There was only one way of making his life happy and comfortable in the Fleet – to get him out of it. But how should they do that? If this question had been asked of every person in London, he would have replied with a laugh: \"Oh, a subscription! to be sure!\"\n",
      "When a gentleman calls on you for anything at all (it may be even only to ask the way to the office) and when it is very much his own business whether you give or not; that gentleman never does so without having an idea of your giving. He always knows that there are two or three ways of getting what he asks, and it never enters into his head but what one of them will be adopted by the person to whom he applies – namely, paying for whatever it may be that is done him. When a gentleman says \"Will you lend me your purse?\" or \"Will you do so much for us?,\" there is always a pretty accurate notion in his mind how he expects to get paid back again; and this notion generally leads him to ask the favour which he asks.\n",
      "There was no doubt in anybody's mind that it was expected Mr Pickwick would make a subscription: but then what could be done by subscription for Mr Pickwick? Nobody had ever heard of making any subscription for anything like that before. All sorts of ways were suggested; all sorts of people talked to on the subject: but nobody could give an answer; and everybody seemed very much inclined to laugh at anybody who should propose giving a subscription in aid of getting Mr Pickwick out of the Fleet.\n",
      "But this only made them work harder; for if it was expected that something would be done for him, somebody or other must do something. The difficulty was to find somebody, and, moreover, that nobody should laugh at anybody else in doing what they did. That was a nice point. They had very little difficulty about the finding part of it: Mr Pickwick was very much talked about, and he was very much read about; his fame and reputation went before him – so that, though every person in London might not have known exactly where he lived (and there were few who did), they all knew more or less what sort of a man Mr Pickwick was. They therefore thought it no very hard matter to find people ready and willing to do anything for Mr Pickwick; and they set about doing something for him, each in his own way and at his own time, without any one person having\n",
      "llama_print_timings:        load time =   12865.96 ms\n",
      "llama_print_timings:      sample time =     575.86 ms /  1024 runs   (    0.56 ms per token,  1778.22 tokens per second)\n",
      "llama_print_timings: prompt eval time =     159.55 ms /   494 tokens (    0.32 ms per token,  3096.25 tokens per second)\n",
      "llama_print_timings:        eval time =   19740.73 ms /  1023 runs   (   19.30 ms per token,    51.82 tokens per second)\n",
      "llama_print_timings:       total time =   20786.15 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "05e70274-169b-4be2-9f7c-553eb99d6361",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1691 (7082d24)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703330250\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 291 tensors from ./models/7B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 4096\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 32\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 11008\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 32\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 32\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - type  f32:   65 tensors\n",
      "llama_model_loader: - type  f16:  226 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 4096\n",
      "llm_load_print_meta: n_head           = 32\n",
      "llm_load_print_meta: n_head_kv        = 32\n",
      "llm_load_print_meta: n_layer          = 32\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 11008\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 7B\n",
      "llm_load_print_meta: model ftype      = F16\n",
      "llm_load_print_meta: model params     = 6.74 B\n",
      "llm_load_print_meta: model size       = 12.55 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size       =    0.11 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: system memory used  =  250.11 MiB\n",
      "llm_load_tensors: VRAM used           = 12603.02 MiB\n",
      "llm_load_tensors: offloading 32 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 33/33 layers to GPU\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 256.00 MB\n",
      "llama_new_context_with_model: KV self size  =  256.00 MiB, K (f16):  128.00 MiB, V (f16):  128.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 676/676\n",
      "llama_new_context_with_model: compute buffer total size = 73.69 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 70.50 MiB\n",
      "llama_new_context_with_model: total VRAM used: 12929.52 MiB (model: 12603.02 MiB, context: 326.50 MiB)\n",
      "\n",
      "system_info: n_threads = 36 / 72 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her Christian pastors she entertained herself, while feasting and dancing, by burning noblemen alive; flocks to the southward of Nismes were devoured; columns of distressed emigrants, fleeing from the execrations of their former servitude, arriving from every quarter of the globe. The guillotine, that national instrument of torture, was fully as extensively employed there as here: for while, in this kingdom, a scoundrel is shown to die every morning at Tyburn; in France, his fellow-criminal, whom we are obliged to call a nobleman, receives (at least until he has paid) the same quality and quantity of torture precisely. \n",
      " The gendarmerie here takes its name from an eminent saint of that persuasion. But as a compensation for the pains and penalties attending that calling in France, they have not, like their brethren here, to plunge into the fire in case of disloyalty; and, moreover, are suffered to retain the fee for their services; an advantage which the gendarmes here do not enjoy. On these accounts, therefore, I could not advise a person to be gendarme who wished to escape persecution from that quarter, or to retire upon a small income. \n",
      "# Chapter XXXI\n",
      "A DETECTION OF THE BROKERS AT THE MARKETS IN THEIR CORRUPTIONS AND EXACTIONS\n",
      "It is the office of the gendarmerie to watch and prevent, as far as it depends on them, those fraudulent dealings which are practised among brokers at markets; but being bribed by the parties, they connive at these abuses and exactions. They take care that no complaints shall be made of this nature to any public authority; but if a person who has been imposed upon, either out of passion or necessity, should be so imprudent as to give them notice, they generally put him into prison without formality until he can find means to make his escape. The gendarme has no other office in such cases than to serve writs for false imprisonment: but it is rare that any of these proceedings ever come to a public trial; the reason whereof seems to be, that those who have the administration of justice at such courts, are likewise themselves the judges and assessors.\n",
      "In markets where they deal in cattle and horses, there is no better way to cheat one's self than to buy what passes for a _nobl_ _é_ , which is only the name of a large horse. As these horses are very dear at five hundred livres (or about eighty pounds), and the small horse not above seventy or eighty pounds, the price seems great, because it includes that of both: and by this device you will be defrauded to the amount of three-fourths part of your money. It is likewise easy to cheat yourself in buying mules; for in buying a small one, you have often an _âne_ , which they call a large one, included with it. A large ass will not be under ninety pounds and sometimes nearer one hundred and twenty: so that the price of three hundred livres includes that of two animals.\n",
      "The manner by which these abuses are carried on, is as follows; there is no buyer or seller of cattle who has not a friend at the market who watches for him and attends upon his business, and who may be trusted with the management of his affairs, particularly in these cases, because it is to his interest that they should be well managed. These men go about at five o'clock in the morning, or half an hour before sunrise, when nobody goes to market but the dealers themselves, who, from habit, are awake so early; and they have no sooner discovered some good cattle which their friends intend to buy, than they hasten to a seller of _nobles_ , and inquire if he has one of that kind. He says, perhaps, yes; but the question is not with him; for the owner may not be willing to part with it. The dealer then goes to a friend of his own who buys from this seller, or from another who keeps a stock of horses and mules in order to sell them at market: if there is no horse to be had that way, they go to some other of these friends; but still they have not one which answers the purpose. They then return with the disappointing news to their first informer, who makes it his business to keep a constant watch for such animals; and if he has not seen any such for two or three days, he goes himself to the s\n",
      "llama_print_timings:        load time =   13114.72 ms\n",
      "llama_print_timings:      sample time =     574.35 ms /  1024 runs   (    0.56 ms per token,  1782.90 tokens per second)\n",
      "llama_print_timings: prompt eval time =     162.53 ms /   494 tokens (    0.33 ms per token,  3039.38 tokens per second)\n",
      "llama_print_timings:        eval time =   19730.40 ms /  1023 runs   (   19.29 ms per token,    51.85 tokens per second)\n",
      "llama_print_timings:       total time =   20778.83 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/7B-v2/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c2c1009-0642-4be0-8fee-f22a3e8bc23a",
   "metadata": {},
   "source": [
    "### 13B Q4_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "134a3c7b-30e2-460c-bfcb-ccb51c090797",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1691 (7082d24)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703330285\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n",
      "llama_model_loader: loaded meta data with 22 key-value pairs and 363 tensors from ./models/13B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 5120\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 40\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 13824\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 40\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 40\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  21:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type q4_0:  281 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 5120\n",
      "llm_load_print_meta: n_head           = 40\n",
      "llm_load_print_meta: n_head_kv        = 40\n",
      "llm_load_print_meta: n_layer          = 40\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 13824\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 13B\n",
      "llm_load_print_meta: model ftype      = Q4_0\n",
      "llm_load_print_meta: model params     = 13.02 B\n",
      "llm_load_print_meta: model size       = 6.86 GiB (4.53 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size       =    0.14 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: system memory used  =   88.03 MiB\n",
      "llm_load_tensors: VRAM used           = 6936.01 MiB\n",
      "llm_load_tensors: offloading 40 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 41/41 layers to GPU\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 400.00 MB\n",
      "llama_new_context_with_model: KV self size  =  400.00 MiB, K (f16):  200.00 MiB, V (f16):  200.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 844/844\n",
      "llama_new_context_with_model: compute buffer total size = 78.19 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 75.00 MiB\n",
      "llama_new_context_with_model: total VRAM used: 7411.01 MiB (model: 6936.01 MiB, context: 475.00 MiB)\n",
      "\n",
      "system_info: n_threads = 36 / 72 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of her Christian pastors, she was furnished with a host of bishops, athletes, contraptions, and steams; while the queen of heaven herself, in the person of the Duchess of Kent, was in the prime of British matronage. Distressing examples of the instability of our own social state were witnessed by all. In London, a vast quantity of ground was built on: streets were lighted with gas: carts and omnibuses became coated with iron: stories were added to houses: and public spaces resounded with the hard footsteps of day-labourers. The eternal Marriage of Thames and Tiber was consummated; after which a long period of years elapsed, during which the whole world (as it did also towards the close of every century) got on wheels: as the consequence of which (also in common with all centuries) some persons mounted omnibuses, and some ran away with the horses. At so irrational a time did London present herself to the world that at the period of which we are speaking there were scarcely twenty thousand houses throughout the entire circuit of her orb. It was a rash era; full of venture, change, and novelty. The exclusive occupation of the land by the rich was not yet disturbed: for their sakes the rest of the nation was poor, or very nearly so. There were a few popular grumbles. The shopkeepers were beginning to be alarmingly numerous: the trading population were every where conspicuously on the increase; and even the old Public Companies had lately become many, in number, and one, in point of magnitude. But they had scarcely taken root at all: their shares (though for a brief season booming) were little better than wild-cat speculations: and there already seemed reason to apprehend that in a few more years they must all inevitably break. There had been two Parliaments, since the close of the last century; and Mr. Pitt was Prime Minister of this realm as of most others, and by common consent.\n",
      "It is probable enough that about this time the new French Revolution first began to make its power felt. It did not take place until some years later: nor had it begun in England, at this present writing, to shake the throne, or menace property: but it was very near doing both. The alarm of such a change (in every part and condition of society) had reached into the remotest corners of the land; and it is recorded of the country town of Staggsbridge that an extraordinary sensation pervaded its whole population, on the sudden apparition, in the public place, of an old man who was universally known to have been a French Revolutionist. It is not too much to say that such was the terror of that name among the multitude, that when this personage had come into full view and was plainly recognised by the crowd as one who had fought at Valmy with General Kluber; and had afterwards emigrated (for his own safety) under the head of a battalion in the King of Prussia's service; even then there was hardly to be found any heart in all the throng which did not palpitate, as it were, with terror.\n",
      "The cause of this public consternation and dismay may best be rendered by extracting from the parish-records certain passages of a curious nature, touching one and another of those who figured most conspicuously in these events.\n",
      "We first take up the Rev. Mr. Veneering, of whom we have spoken on some former occasion (in connexion with his celebrated sermon entitled The Duty of Self-Denial). This reverend gentleman was very much struck indeed with the unlooked-for occurrence just referred to. So much so, that he took up a pen and wrote to the public press upon it; wherein he expressed himself as being grieved beyond measure at this open profession of sentiments subversive of the rights of property (amongst which, we beg leave to add, were the rectorial tithes of Staggsbridge), and so on, in a very edifying manner. But it happened that, even while Mr. Veneering's article was still unpublished, and was only lying at his own house ready for publication; a most extraordinary circumstance came to pass at the same place (Staggsbridge), which caused this reverend gentleman to pause in his career of zeal against those opinions he held so reprehensible.\n",
      "Mr. Veneering's father had been a butcher, who had risen by means of a will, and in some mysterious way unascertained to the present hour, to be the owner of considerable property. This property was\n",
      "llama_print_timings:        load time =    6624.85 ms\n",
      "llama_print_timings:      sample time =     575.86 ms /  1024 runs   (    0.56 ms per token,  1778.20 tokens per second)\n",
      "llama_print_timings: prompt eval time =     298.88 ms /   494 tokens (    0.61 ms per token,  1652.83 tokens per second)\n",
      "llama_print_timings:        eval time =   13619.27 ms /  1023 runs   (   13.31 ms per token,    75.11 tokens per second)\n",
      "llama_print_timings:       total time =   14803.93 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/13B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "aad047cf-d4d1-4e14-937f-1dd1e456a11b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1691 (7082d24)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703330308\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n",
      "llama_model_loader: loaded meta data with 22 key-value pairs and 363 tensors from ./models/13B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 5120\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 40\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 13824\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 40\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 40\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  21:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type q4_0:  281 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 5120\n",
      "llm_load_print_meta: n_head           = 40\n",
      "llm_load_print_meta: n_head_kv        = 40\n",
      "llm_load_print_meta: n_layer          = 40\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 13824\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 13B\n",
      "llm_load_print_meta: model ftype      = Q4_0\n",
      "llm_load_print_meta: model params     = 13.02 B\n",
      "llm_load_print_meta: model size       = 6.86 GiB (4.53 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size       =    0.14 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: system memory used  =   88.03 MiB\n",
      "llm_load_tensors: VRAM used           = 6936.01 MiB\n",
      "llm_load_tensors: offloading 40 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 41/41 layers to GPU\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 400.00 MB\n",
      "llama_new_context_with_model: KV self size  =  400.00 MiB, K (f16):  200.00 MiB, V (f16):  200.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 844/844\n",
      "llama_new_context_with_model: compute buffer total size = 78.19 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 75.00 MiB\n",
      "llama_new_context_with_model: total VRAM used: 7411.01 MiB (model: 6936.01 MiB, context: 475.00 MiB)\n",
      "\n",
      "system_info: n_threads = 36 / 72 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of its noble author, the British nation did the like; and thus preferring present ease and comfort to the injury of future generations, fell into a penitential state. Bankrupt in soul and body, Man had no diagnosis left but to hear great talk about the Deity from first rate divines, who assured him that all was vanity. A miserable picture it is if they are correct. Yet so it seems to be. Highly unnatural as the doctrine will appear, it still is an established fact that there is no faith in the possible future of human affairs; and that although the present generation is now in the zenith of its light upon this planet, yet one thousand millions of ages hence will find it not at its decline, but at midnight. The night comes when Man goes to his long rest. \n",
      " BOOK THE FIRST—SUNSET\n",
      " Chapter One  \n",
      " Five Years Later\n",
      " Five years, a month, and a day had elapsed since Mrs. Darnay had stood beside the bed where the prisoner, the young girl whom he loved, and he, lay under sentence of death for their lives. \n",
      " Five years, a month, and a day! What strange things change when the hand is on the dial of Time; and the lightest finger which traces it around detects a difference in its revolving circuit, greater than lies between the rind and heart of an orange! \n",
      " The prison had long ago been razed to the ground. All the grown-up figures of the trial were gone, excepting one alone—a prisoner doomed to die within the shadow of another and a higher guillotine, which was also broken down. He sat in his condemned cell at the Conciergerie when that month, and day, and year rolled round; and in its going out saw the last sunset light up the western windows of La Force. \n",
      " It was in this place of confinement, with these objects before him, that Charles Darnay sat when night was closing in again. There was no noise in his cell but the sound of his own footfall as he paced to and fro in it: nor any light in it save what struggled through the iron wires of a barred window high up in the wall; and by this feeble ray he read, for the hundredth time, two letters—one beginning 'My very dear,' and the other ending with these words, 'Your unhappy Lucie.' He held the hand-written notes to his lips. 'She kisses them a thousand times,' he said, and pressed them passionately to his heart. \n",
      " The light failed him gradually for another hour; the gloom closed in upon him, the iron grating of its barred window seemed to close upon him with it; but, with an eager anticipation and quickening pulse, he remembered that at this time the gates of the prison would be opened, and that at this time Lucie came. \n",
      " The door was unbarred, the outer door was slowly pushed back by some heavy hand. A figure entered, and advanced to where he stood; a light shone in. \n",
      " It was not the gaoler; it was the Conciergerie, with its own turnkey—Death. \n",
      " The prisoner did not start or shrink as he looked on Death: they were friends. 'I am ready,' said Charles Darnay; 'let us go on to the end.' He smiled and laughed, and cried, in a low voice, as he had cried many times before, when talking to her of his coming death—'Lucie! Lucie! Lucie!' \n",
      " The light passed from him as he spoke these words, for the Conciergerie was but a dark vault. The last gleam flickered and went out; he groped with his hands before him in its dense gloom. There came another voice from afar off in the night, very low and sweet. He knew it well: 'Darnay! Darnay!'\n",
      "She was there, in his dungeon at the dead of night—a face above an angel's would have been no fairer sight; but he could not see her. \n",
      " His arm stole round her; his lips met hers; his voice whispered to her; a minute more, and it would be forever still. There was a pause in the street outside: footsteps were audible upon its stones: steps were approaching nearer. They had come to fetch him to execution—the victim at the block. \n",
      " A key grated in the lock of his dungeon door; it opened, and admitted a ray from an adjoining room. In came two figures with their lamps. He fell away from Lucie with a cry of agony. 'The lamp!' he said\n",
      "llama_print_timings:        load time =    6212.61 ms\n",
      "llama_print_timings:      sample time =     574.59 ms /  1024 runs   (    0.56 ms per token,  1782.13 tokens per second)\n",
      "llama_print_timings: prompt eval time =     301.44 ms /   494 tokens (    0.61 ms per token,  1638.80 tokens per second)\n",
      "llama_print_timings:        eval time =   13708.58 ms /  1023 runs   (   13.40 ms per token,    74.62 tokens per second)\n",
      "llama_print_timings:       total time =   14894.29 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/13B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "bb28eb3a-3cee-45c0-b25b-ad68ad8bdcf5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1691 (7082d24)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703330330\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n",
      "llama_model_loader: loaded meta data with 22 key-value pairs and 363 tensors from ./models/13B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 5120\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 40\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 13824\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 40\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 40\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  21:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type q4_0:  281 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 5120\n",
      "llm_load_print_meta: n_head           = 40\n",
      "llm_load_print_meta: n_head_kv        = 40\n",
      "llm_load_print_meta: n_layer          = 40\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 13824\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 13B\n",
      "llm_load_print_meta: model ftype      = Q4_0\n",
      "llm_load_print_meta: model params     = 13.02 B\n",
      "llm_load_print_meta: model size       = 6.86 GiB (4.53 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size       =    0.14 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: system memory used  =   88.03 MiB\n",
      "llm_load_tensors: VRAM used           = 6936.01 MiB\n",
      "llm_load_tensors: offloading 40 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 41/41 layers to GPU\n",
      "...................................................................................................\n",
      "llama_new_context_with_model: n_ctx      = 512\n",
      "llama_new_context_with_model: freq_base  = 10000.0\n",
      "llama_new_context_with_model: freq_scale = 1\n",
      "llama_kv_cache_init: VRAM kv self = 400.00 MB\n",
      "llama_new_context_with_model: KV self size  =  400.00 MiB, K (f16):  200.00 MiB, V (f16):  200.00 MiB\n",
      "llama_build_graph: non-view tensors processed: 844/844\n",
      "llama_new_context_with_model: compute buffer total size = 78.19 MiB\n",
      "llama_new_context_with_model: VRAM scratch buffer: 75.00 MiB\n",
      "llama_new_context_with_model: total VRAM used: 7411.01 MiB (model: 6936.01 MiB, context: 475.00 MiB)\n",
      "\n",
      "system_info: n_threads = 36 / 72 | AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n",
      "sampling: \n",
      "\trepeat_last_n = 64, repeat_penalty = 1.100, frequency_penalty = 0.000, presence_penalty = 0.000\n",
      "\ttop_k = 40, tfs_z = 1.000, top_p = 0.950, min_p = 0.050, typical_p = 1.000, temp = 1.100\n",
      "\tmirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000\n",
      "sampling order: \n",
      "CFG -> Penalties -> top_k -> tfs_z -> typical_p -> top_p -> min_p -> temp \n",
      "generate: n_ctx = 512, n_batch = 512, n_predict = 1024, n_keep = 0\n",
      "\n",
      "\n",
      "\u001b[33m It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. \n",
      " There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. \n",
      " It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. \n",
      " France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\u001b[0m Under the guidance of its noble architect, Monsieur François-Arago, that most sublime of all human inventions, the barometer, had long ago indicated to the people of Paris and Lyons that the time was at hand when, if not already come, the elements would be found inconveniently close upon their shoulders. Instantly, like an army pouring out of its works, they stood beneath the showers, waiting for them; until the police regulated omnibuses, and the water-drinkers. Meanwhile, the press went on, in its superfluous manner, budding forth newspapers, magazines, pamphlets, swaddling-bands of every size, as though the clouds would never open. Opened at last – a stream of refugees pouring like an army pouring out from the soil of France, and the earth drank their blood! \n",
      " It is a far, far better thing that I do, than I have ever done; it is a far, far better rest that I go to than I have ever known.\n",
      "# BOOK THE FIRST: SIX YEARS\n",
      "CHAPTER II – TOMMY TRADDLES\n",
      "In which Tommy Traddles and the Doctor play at catching, and have their hands tied by the knots of an ancient friendship\n",
      "I have no reason to suppose that any other mortal ever lived who did not in his childhood make for himself, out of all sorts of queer scraps, a personal image. Perhaps this is the same as what they call fetish in some of the black tribes. And I believe it is also true that all little children like to play with dolls; and it might be as well worth while to try to discover why it should be so, even though you never did find out. But whether the impulse to make a fetish or doll arises from a need to animate the things of earth, or from the delight that human creatures have in seeing them so animated, I will not pretend to know; nor can I explain why my personal fetish, which was also my dearest friend and playmate, should be neither boy nor girl.\n",
      "For I never had a sister or brother; but when I was still very young indeed (my father used to tell me that I could scarcely walk) I made myself a little puppet of wool, with a head like the heads of babies cut out of penny prints in the shops – which penny prints are to be seen and bought even now. And this was my friend and playfellow; and it had eyes that opened and shut, but could not see nor hear nor speak. I named it Jip.\n",
      "But my mother would not allow me to have my puppet at meals; and she made a great point of cleanliness, in which she used to make me very unhappy indeed. For I loved my little doll so dearly, and was never without him when awake, that if ever by any mischance he fell into the fire or downstairs or into the pond, I would weep over his ruined image and repentance as bitterly as though I myself had fallen into the fire or downstays. But my mother made me wash him when he fell into the water; which was indeed hard on both of us; since how could I look at Jip again if once I knew that it was wet? So there was no end to the quarrels between Jip and I, whenever his turn came to be washed. But as this always ended in Jip's being made dry by the kitchen fire (for we had only one) it seemed to me that it was better even if Jip did not like to go through water.\n",
      "For when all is done, there are none of us so foolish but what we can have a friend, if we will look for one; and many of us would find more joy than sorrow if our own children were to tell us now and again, \"Mother (or Father) dear, I have found somebody who wants to be a friend of mine.\"\n",
      "For there are friends enough in the world, as well as enemies. But we do not know how to choose our friends, and it seems that even those that we try to choose for ourselves never will suit us if they may.\n",
      "There was a gentleman called Mr. Hands who lived on the outskirts of the town where my master went to school. This Mr. Hands was very fond of animals; especially he loved cats and dogs, and all that kind of thing. And at the end of our garden was his paddock for his favourite donkeys. I never could find out why they were called donkies instead of donkeys; but they used to call them so because Mr. Hands was their owner, which is what we say in this town\n",
      "llama_print_timings:        load time =    3506.25 ms\n",
      "llama_print_timings:      sample time =     577.97 ms /  1024 runs   (    0.56 ms per token,  1771.71 tokens per second)\n",
      "llama_print_timings: prompt eval time =     296.30 ms /   494 tokens (    0.60 ms per token,  1667.26 tokens per second)\n",
      "llama_print_timings:        eval time =   13705.57 ms /  1023 runs   (   13.40 ms per token,    74.64 tokens per second)\n",
      "llama_print_timings:       total time =   14890.76 ms\n",
      "Log end\n"
     ]
    }
   ],
   "source": [
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/13B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b4d1552a-9361-4433-aa57-b616ecd13dee",
   "metadata": {},
   "source": [
    "### 13B f16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "24e9abc3-f9fc-4faa-9d60-d8971e686e69",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1691 (7082d24)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703330350\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n",
      "llama_model_loader: loaded meta data with 21 key-value pairs and 363 tensors from ./models/13B-v2/ggml-model-f16.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 5120\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 40\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 13824\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 40\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 40\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 1\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - type  f32:   81 tensors\n",
      "llama_model_loader: - type  f16:  282 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 5120\n",
      "llm_load_print_meta: n_head           = 40\n",
      "llm_load_print_meta: n_head_kv        = 40\n",
      "llm_load_print_meta: n_layer          = 40\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 1\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 13824\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 13B\n",
      "llm_load_print_meta: model ftype      = F16\n",
      "llm_load_print_meta: model params     = 13.02 B\n",
      "llm_load_print_meta: model size       = 24.24 GiB (16.00 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size       =    0.14 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: system memory used  =  312.64 MiB\n",
      "llm_load_tensors: VRAM used           = 24514.08 MiB\n",
      "llm_load_tensors: offloading 40 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 41/41 layers to GPU\n",
      ".................................................................................................\n",
      "CUDA error 2 at ggml-cuda.cu:9081: out of memory\n",
      "current device: 0\n",
      "GGML_ASSERT: ggml-cuda.cu:9081: !\"CUDA error\"\n"
     ]
    }
   ],
   "source": [
    "# Out of memory\n",
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/13B-v2/ggml-model-f16.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04edde69-2bb1-4cd2-b0af-e265f93f128b",
   "metadata": {},
   "source": [
    "### 70B Q4_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0aac592f-7a5c-41b1-88f5-f4d3157af8b1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log start\n",
      "main: build = 1691 (7082d24)\n",
      "main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\n",
      "main: seed  = 1703330372\n",
      "ggml_init_cublas: GGML_CUDA_FORCE_MMQ:   no\n",
      "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n",
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6\n",
      "llama_model_loader: loaded meta data with 22 key-value pairs and 723 tensors from ./models/70B-v2/ggml-model-q4_0.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = llama\n",
      "llama_model_loader: - kv   1:                               general.name str              = LLaMA v2\n",
      "llama_model_loader: - kv   2:                       llama.context_length u32              = 4096\n",
      "llama_model_loader: - kv   3:                     llama.embedding_length u32              = 8192\n",
      "llama_model_loader: - kv   4:                          llama.block_count u32              = 80\n",
      "llama_model_loader: - kv   5:                  llama.feed_forward_length u32              = 28672\n",
      "llama_model_loader: - kv   6:                 llama.rope.dimension_count u32              = 128\n",
      "llama_model_loader: - kv   7:                 llama.attention.head_count u32              = 64\n",
      "llama_model_loader: - kv   8:              llama.attention.head_count_kv u32              = 8\n",
      "llama_model_loader: - kv   9:     llama.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
      "llama_model_loader: - kv  10:                          general.file_type u32              = 2\n",
      "llama_model_loader: - kv  11:                       tokenizer.ggml.model str              = llama\n",
      "llama_model_loader: - kv  12:                      tokenizer.ggml.tokens arr[str,32000]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
      "llama_model_loader: - kv  13:                      tokenizer.ggml.scores arr[f32,32000]   = [0.000000, 0.000000, 0.000000, 0.0000...\n",
      "llama_model_loader: - kv  14:                  tokenizer.ggml.token_type arr[i32,32000]   = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.merges arr[str,61249]   = [\"▁ t\", \"e r\", \"i n\", \"▁ a\", \"e n...\n",
      "llama_model_loader: - kv  16:                tokenizer.ggml.bos_token_id u32              = 1\n",
      "llama_model_loader: - kv  17:                tokenizer.ggml.eos_token_id u32              = 2\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 0\n",
      "llama_model_loader: - kv  19:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  20:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  21:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - type  f32:  161 tensors\n",
      "llama_model_loader: - type q4_0:  561 tensors\n",
      "llama_model_loader: - type q6_K:    1 tensors\n",
      "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n",
      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
      "llm_load_print_meta: arch             = llama\n",
      "llm_load_print_meta: vocab type       = SPM\n",
      "llm_load_print_meta: n_vocab          = 32000\n",
      "llm_load_print_meta: n_merges         = 0\n",
      "llm_load_print_meta: n_ctx_train      = 4096\n",
      "llm_load_print_meta: n_embd           = 8192\n",
      "llm_load_print_meta: n_head           = 64\n",
      "llm_load_print_meta: n_head_kv        = 8\n",
      "llm_load_print_meta: n_layer          = 80\n",
      "llm_load_print_meta: n_rot            = 128\n",
      "llm_load_print_meta: n_gqa            = 8\n",
      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
      "llm_load_print_meta: n_ff             = 28672\n",
      "llm_load_print_meta: n_expert         = 0\n",
      "llm_load_print_meta: n_expert_used    = 0\n",
      "llm_load_print_meta: rope scaling     = linear\n",
      "llm_load_print_meta: freq_base_train  = 10000.0\n",
      "llm_load_print_meta: freq_scale_train = 1\n",
      "llm_load_print_meta: n_yarn_orig_ctx  = 4096\n",
      "llm_load_print_meta: rope_finetuned   = unknown\n",
      "llm_load_print_meta: model type       = 70B\n",
      "llm_load_print_meta: model ftype      = Q4_0\n",
      "llm_load_print_meta: model params     = 68.98 B\n",
      "llm_load_print_meta: model size       = 36.20 GiB (4.51 BPW) \n",
      "llm_load_print_meta: general.name     = LLaMA v2\n",
      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
      "llm_load_print_meta: EOS token        = 2 '</s>'\n",
      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
      "llm_load_tensors: ggml ctx size       =    0.28 MiB\n",
      "llm_load_tensors: using CUDA for GPU acceleration\n",
      "llm_load_tensors: system memory used  =  140.90 MiB\n",
      "llm_load_tensors: VRAM used           = 36930.11 MiB\n",
      "llm_load_tensors: offloading 80 repeating layers to GPU\n",
      "llm_load_tensors: offloading non-repeating layers to GPU\n",
      "llm_load_tensors: offloaded 81/81 layers to GPU\n",
      "................................................................\n",
      "CUDA error 2 at ggml-cuda.cu:9081: out of memory\n",
      "current device: 0\n",
      "GGML_ASSERT: ggml-cuda.cu:9081: !\"CUDA error\"\n"
     ]
    }
   ],
   "source": [
    "# Out of memory\n",
    "!./main --color --no-mmap -ngl 10000 --temp 1.1 --repeat_penalty 1.1 -n 1024 --ignore-eos -m ./models/70B-v2/ggml-model-q4_0.gguf  -p \"It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way – in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. <0x0A>\\\n",
    "There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. <0x0A>\\\n",
    "It was the year of Our Lord one thousand seven hundred and seventy-five. Spiritual revelations were conceded to England at that favoured period, as at this. Mrs. Southcott had recently attained her five-and-twentieth blessed birthday, of whom a prophetic private in the Life Guards had heralded the sublime appearance by announcing that arrangements were made for the swallowing up of London and Westminster. Even the Cock-lane ghost had been laid only a round dozen of years, after rapping out its messages, as the spirits of this very year last past (supernaturally deficient in originality) rapped out theirs. Mere messages in the earthly order of events had lately come to the English Crown and People, from a congress of British subjects in America: which, strange to relate, have proved more important to the human race than any communications yet received through any of the chickens of the Cock-lane brood. <0x0A>\\\n",
    "France, less favoured on the whole as to matters spiritual than her sister of the shield and trident, rolled with exceeding smoothness down hill, making paper money and spending it.\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
